{"meta":{"date":"2026-10-28","timezone":"UTC","generated_at":"2026-10-28T23:39:28Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":75,"failed_runs":52,"runs_with_occurrences":51,"runs_with_failure_pattern_matches":40,"runs_with_unmatched_occurrences":11,"runs_without_occurrence_rows":24,"failed_runs_without_occurrence_rows":1},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115574283206922240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"2b27b338555f48535f6e6f9cd438741324be4e58","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T22:39:49Z","started_at":"2026-10-28T22:39:49Z","completed_at":"2026-10-28T22:40:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T22:49:01.009017814Z","timing_metadata_checked_at":"2026-10-28T22:49:01.009017814Z","region_metadata_state":"pending","region_metadata_first_checked_at":"2026-10-28T22:49:01.016503491Z","region_metadata_checked_at":"2026-10-28T23:21:19.481935407Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"failed_without_raw_rows"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115560273719857152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"6405f41e9d626eac01c2304a4b0fb1eefd6c1e81","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T21:44:09Z","started_at":"2026-10-28T21:44:09Z","completed_at":"2026-10-28T22:52:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T23:10:35.542284698Z","timing_metadata_checked_at":"2026-10-28T23:10:35.542284698Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T23:10:35.548934679Z","region_metadata_checked_at":"2026-10-28T23:10:35.548934679Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3f22e8c2b15ddd83eaf64646471723dbd20879eeb73c6423804ffb3b693489a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115560273719857152","occurred_at":"2026-10-28T21:44:09Z","failed_at":"provision","signature_id":"6c11c44592820325fa1934724e67fb473057c5c4359853d87ec5141e3481ef08","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T22:46:09.268Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 description=\"Step nsp\\n Kind: ARM\\n Template: templates/mgmt-nsp.bicep\\n Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-28T22:46:09.760Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1\ntime=2026-10-28T22:46:10.194Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 deployment=351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9857152-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2\ntime=2026-10-28T22:47:10.427Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Resources/deployments/351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2/operationStatuses/08584083793155131248\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Resources/deployments/profile-ufpn5r2pwmb4i\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: 824ad833-5117-4ca4-a2e6-1002e46997d8\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9857152-mgmt-1/resourceAssociations/nsp-j9857152-mgmt-1-ra-67927217-e211-550f-abbf-b3e73881a01a\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AssociationFailedAtResourceProvider\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Association provisioning failed at resource provider\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75e25b16c89c5bd443d1560c5ffb0c0789ec4d49846a07583c802aab771d1e1c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AssociationFailedAtResourceProvider; detail message Association provisioning failed at resource provider; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75e25b16c89c5bd443d1560c5ffb0c0789ec4d49846a07583c802aab771d1e1c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7817/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115537945669668864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7817,"pr_state":"open","pr_sha":"8bf5e0dd8f3ecdfb03900def5da93c7f0e47909d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-28T20:15:16Z","started_at":"2026-10-28T20:15:16Z","completed_at":"2026-10-28T22:28:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T22:38:25.69494507Z","timing_metadata_checked_at":"2026-10-28T22:38:25.69494507Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T22:38:25.69799816Z","region_metadata_checked_at":"2026-10-28T22:38:25.69799816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115536999510183936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7809,"pr_state":"open","pr_sha":"2a5af5f0f25085953117ef4c51755e96d1aa23bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T20:11:40Z","started_at":"2026-10-28T20:11:40Z","completed_at":"2026-10-28T22:22:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T22:38:25.548938546Z","timing_metadata_checked_at":"2026-10-28T22:38:25.548938546Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T22:38:25.562552101Z","region_metadata_checked_at":"2026-10-28T22:38:25.562552101Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"09df32c648a1126df5b1a42cb95d5fa1959c9b7d4c53d94e2bb870d735b8b242","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115536999510183936","occurred_at":"2026-10-28T20:11:40Z","failed_at":"e2e","signature_id":"ea34188856e004cdd568b90c2383e830d4cf10e6aa357acf563621e40e9840a2","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x4218f58d8c0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8q6szjwkq6rk\" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n \"name\": \"78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T21:13:11.691142567Z\",\n \"endTime\": \"2026-10-28T21:32:17.541374372Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d\n client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x4218f58d8c0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8q6szjwkq6rk\" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n \"name\": \"78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T21:13:11.691142567Z\",\n \"endTime\": \"2026-10-28T21:32:17.541374372Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d\n client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0a3bf853ee2796c58618727ae662e023afe77aeea1b8cb1ea78a9c68517783b0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional t... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6b14bf5415e7f0230be683b90fadd80672f221f8d51c69c1939b303149998eaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115536999510183936","occurred_at":"2026-10-28T20:11:40Z","failed_at":"e2e","signature_id":"8507857a28426cbd2659cf220f0916407db048853ec46c983b6852d107a04737","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2424493382e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x2424493345b0\u003e{\n s: \"dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2424493382e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x2424493345b0\u003e{\n s: \"dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e56d0b8c2e490620d82d524d28d93e57a3cdbe4dd0316b01ad0ff503a0a27508","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0a3bf853ee2796c58618727ae662e023afe77aeea1b8cb1ea78a9c68517783b0","e56d0b8c2e490620d82d524d28d93e57a3cdbe4dd0316b01ad0ff503a0a27508"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533269758906368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7757,"pr_state":"open","pr_sha":"75e7373f66b7c971bb6ed077b853fa76e9147016","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T19:56:50Z","started_at":"2026-10-28T19:56:50Z","completed_at":"2026-10-28T22:13:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T22:28:05.109268103Z","timing_metadata_checked_at":"2026-10-28T22:28:05.109268103Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T22:28:05.112753792Z","region_metadata_checked_at":"2026-10-28T22:28:05.112753792Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6af875f1f2ad5024a27be34122bd1cf8a0ca8608096279be608e679f3a4f0774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533269758906368","occurred_at":"2026-10-28T19:56:50Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T21:09:13Z\n Ended: 2026-10-28T21:26:12Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8906368-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"f7b804a5c371b51adbcc8fb8c3c930ed02dbcdb76adc1dff0cb2da51ad4a0800","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533269758906368","occurred_at":"2026-10-28T19:56:50Z","failed_at":"e2e","signature_id":"0e25a1ce387e1311f273f459aa16bea882049776f82a3cf61fd90170dc6dacb5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-7tbfp9\"\nUnexpected error:\n \u003c*errors.joinError | 0x28672532e420\u003e: \n VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-bkpxxt\" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-7tbfp9\"\nUnexpected error:\n \u003c*errors.joinError | 0x28672532e420\u003e: \n VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-bkpxxt\" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c68cfdadf7a813f7306833771fe3f1479e7865cf15d87626b36ad658cb57e3bf","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-bkpxxt' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","c68cfdadf7a813f7306833771fe3f1479e7865cf15d87626b36ad658cb57e3bf"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533209771970560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7693,"pr_state":"open","pr_sha":"0bb0b65e8a0937e1c8c0438e405c5860e7716e58","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T19:56:36Z","started_at":"2026-10-28T19:56:36Z","completed_at":"2026-10-28T20:19:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:31:23.878456026Z","timing_metadata_checked_at":"2026-10-28T20:31:23.878456026Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T20:31:23.882160715Z","region_metadata_checked_at":"2026-10-28T21:35:05.994158323Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3b088d7f25cd74bb0ee62a882357056fdb47005e075488b6f030788c2bad9e87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533209771970560","occurred_at":"2026-10-28T19:56:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115528237315526656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7787,"pr_state":"closed","pr_sha":"ffe9a1a535afaeb4ceb215e0f27b578fa90f9ded","final_merged_sha":"ffe9a1a535afaeb4ceb215e0f27b578fa90f9ded","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T19:36:51Z","started_at":"2026-10-28T19:36:51Z","completed_at":"2026-10-28T21:45:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:56:36.856581135Z","timing_metadata_checked_at":"2026-10-28T21:56:36.856581135Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:56:36.860383123Z","region_metadata_checked_at":"2026-10-28T21:56:36.860383123Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115527125657194496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7651,"pr_state":"open","pr_sha":"0fbb7c02d89394d51ddeb8b7a468df16887a224e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T19:32:25Z","started_at":"2026-10-28T19:32:25Z","completed_at":"2026-10-28T20:48:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:24:08.937766251Z","timing_metadata_checked_at":"2026-10-28T21:24:08.937766251Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:24:08.941605241Z","region_metadata_checked_at":"2026-10-28T21:24:08.941605241Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cc3bc907382ab0f88d7dcccbc0db4890bd942b20171203b84adada61fd4ed4a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115527125657194496","occurred_at":"2026-10-28T19:32:25Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7666,"pr_state":"open","pr_sha":"f46d8ee64f260f2d90f3e9da88f853c979dcfa61","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T19:15:51Z","started_at":"2026-10-28T19:15:51Z","completed_at":"2026-10-28T21:48:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:56:36.653539154Z","timing_metadata_checked_at":"2026-10-28T21:56:36.653539154Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:56:36.656665044Z","region_metadata_checked_at":"2026-10-28T21:56:36.656665044Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0a8fcb04ad068103015dceee4fc75b2e770e3c310879302ae8b74b71f0782a93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","failed_at":"alert","signature_id":"648e77fd094ec15da0a086a524b7c1d4b0cc0eb05bbe4ba4df4fe5e22225b252","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T20:37:24Z\n Ended: 2026-10-28T21:04:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T20:37:24Z\n Ended: 2026-10-28T21:01:24Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"3c3425bc5be5f1dadebf00c8ea746764aaa28de77ef0d572eb9a0591407fc6a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","failed_at":"alert","signature_id":"22cb4c1c74c288852b46936e29daa10fb7e56baf99940791080b2bc27593091c","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T20:39:20Z\n Ended: 2026-10-28T21:05:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"777a1913-78dc-4e11-9844-4f50d6675327\"\n Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-28T20:40:18Z\n Ended: 2026-10-28T21:06:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"777a1913-78dc-4e11-9844-4f50d6675327\"\n Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"45be9c14bcd07de9a3059dc0afa5fd4172ea6c85522d122baf8d4286cd48e7ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","failed_at":"alert","signature_id":"721f8230921d7beffc6045136cc6ccc09c8b2023501ae51dd3d280a96a067666","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T20:52:20Z\n Ended: 2026-10-28T21:03:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T20:52:20Z\n Ended: 2026-10-28T21:03:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"73bbecacf6b78f8ab902bb76dfdc020189b5b0dd10bc7302c2413894a69bc282","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","failed_at":"e2e","signature_id":"c5700ceb14bbf3145f2fe9d4d95f47c4875ea9e5983e09584bafc3d75fbf7935","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2a28c8a260\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h8dst6v865z6\" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n \"name\": \"455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:16:02.73250898Z\",\n \"endTime\": \"2026-10-28T20:45:04.251585998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n correlation ID: 582a61df-2172-42e6-962b-b544533d3da6\n client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2a28c8a260\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h8dst6v865z6\" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n \"name\": \"455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:16:02.73250898Z\",\n \"endTime\": \"2026-10-28T20:45:04.251585998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n correlation ID: 582a61df-2172-42e6-962b-b544533d3da6\n client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115521694461333504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7802,"pr_state":"closed","pr_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","final_merged_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T19:10:49Z","started_at":"2026-10-28T19:10:49Z","completed_at":"2026-10-28T21:39:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:45:54.584389372Z","timing_metadata_checked_at":"2026-10-28T21:45:54.584389372Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:45:54.587917061Z","region_metadata_checked_at":"2026-10-28T21:45:54.587917061Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115521666367885312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7809,"pr_state":"open","pr_sha":"2a5af5f0f25085953117ef4c51755e96d1aa23bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T19:10:44Z","started_at":"2026-10-28T19:10:44Z","completed_at":"2026-10-28T19:38:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:49:07.156175755Z","timing_metadata_checked_at":"2026-10-28T19:49:07.156175755Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T19:49:07.16859032Z","region_metadata_checked_at":"2026-10-28T20:52:21.133163263Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"50e7d03bf6a12cfa7b91ee7be51d41d42b691cefbb8675077e704af3d4e64033","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115521666367885312","occurred_at":"2026-10-28T19:10:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7764,"pr_state":"open","pr_sha":"84f443c3eed5271c273cbdd5de7544b2c41e5f73","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T18:33:36Z","started_at":"2026-10-28T18:33:36Z","completed_at":"2026-10-28T20:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:52:21.040663149Z","timing_metadata_checked_at":"2026-10-28T20:52:21.040663149Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:52:21.043883739Z","region_metadata_checked_at":"2026-10-28T20:52:21.043883739Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"22bccf7550bc9b136abb0fb96ffb9190f17985ffa3f14c7484e9b83e62e2ac7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"alert","signature_id":"87b5b22f77bfe3999c0381b1dc3ddb0ccc1c296599755520fd01de4f586c3961","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T20:03:50Z\n Ended: 2026-10-28T20:15:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T20:03:50Z\n Ended: 2026-10-28T20:15:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"4bd374e2493e01a9d4d2ce80a7c7dd85f3004854f2d00d79312f66903ac0afe1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"alert","signature_id":"2b3dcdce11438bf15723387608c36e111821479ddc9eeeea7696800a37119f7d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T19:48:54Z\n Ended: 2026-10-28T20:14:47Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T19:48:54Z\n Ended: 2026-10-28T20:14:47Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"5969c58e9663d680f933fdf1fbd44ad44dd62baf72dba3327344ab196640f37a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"alert","signature_id":"da5e4122e4c53523b9d3713667cdf04b91bd9de900409d5886b3e4d9e6217e9e","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T19:50:44Z\n Ended: 2026-10-28T20:18:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7\"\n Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-28T19:50:44Z\n Ended: 2026-10-28T20:19:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7\"\n Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"6deac8e7d3f6b9970a83821cd89b21685cd0f7ef939c8cdb8016b698e883e4c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"e2e","signature_id":"60a0140959ffc2ca32cabe626c237c34f8d6c30d556a56d5f6cfe1fa2ef7e592","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x370d5101eb20\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tgjxrh8mcrw6\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\",\n \"name\": \"cce1648d-4b31-48bb-9884-f170bb5391eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:55.409225524Z\",\n \"endTime\": \"2026-10-28T19:58:03.222551854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e\n client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x370d5101eb20\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tgjxrh8mcrw6\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\",\n \"name\": \"cce1648d-4b31-48bb-9884-f170bb5391eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:55.409225524Z\",\n \"endTime\": \"2026-10-28T19:58:03.222551854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e\n client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7b9c25ea180b27c8392a3842f9bcafd533586502103e9343e12677d2030f147e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"e2e","signature_id":"1a2246634d007df6ee517467969a7df4a5c707e703c29c130bd6ae72b247e256","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-678df7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d18a6e97e00\u003e: \n DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d18a6c74540\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-678df7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d18a6e97e00\u003e: \n DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d18a6c74540\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"fcb3ddfc944f2acd252a63a0b0b7018be9b1fc29021b64ae556923240b43a7e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","failed_at":"e2e","signature_id":"68ad6dfac63b490246484162a2cfeb7d64d4970cbbabdb4db8652e41011fa267","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x740ee223cf8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-dmmfjzfldpm8\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\",\n \"name\": \"e51bb1e4-4996-454f-a81d-9354613a75ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:36:46.639467207Z\",\n \"endTime\": \"2026-10-28T19:55:53.152282953Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-8z842-x96hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-8z842-tlsx7: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-dmmfjzfldpm8--managed/cilium-cl-cilium-np-8z842-tlsx7-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-dmmfjzfldpm8--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-8z842-tlsx7-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1d2f4929-515e-4e8d-8580-836e819f8d50' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n correlation ID: 4963ae88-5109-4671-873f-f28a9deccdac\n client request ID: 38a19ead-d2a2-4130-9e04-1364166950d2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x740ee223cf8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-dmmfjzfldpm8\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\",\n \"name\": \"e51bb1e4-4996-454f-a81d-9354613a75ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:36:46.639467207Z\",\n \"endTime\": \"2026-10-28T19:55:53.152282953Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-8z842-x96hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-8z842-tlsx7: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-dmmfjzfldpm8--managed/cilium-cl-cilium-np-8z842-tlsx7-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-dmmfjzfldpm8--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-8z842-tlsx7-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1d2f4929-515e-4e8d-8580-836e819f8d50' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n correlation ID: 4963ae88-5109-4671-873f-f28a9deccdac\n client request ID: 38a19ead-d2a2-4130-9e04-1364166950d2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7816/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512185487626240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7816,"pr_state":"open","pr_sha":"46a7b9567924dd1873a0e3b2f67d6485345374ae","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T18:33:03Z","started_at":"2026-10-28T18:33:03Z","completed_at":"2026-10-28T20:54:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:03:21.014811362Z","timing_metadata_checked_at":"2026-10-28T21:03:21.014811362Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:03:21.019685946Z","region_metadata_checked_at":"2026-10-28T21:03:21.019685946Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a49002bf9cd59dfac40b7c5af97479f17038ac09fd71ac3b31a584773fc3e107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7816/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512185487626240","occurred_at":"2026-10-28T18:33:03Z","failed_at":"e2e","signature_id":"eed1cf90a87829b734d34d65add9b5f9a3ad3af46df74a86ca0e186f05eb617f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f06fbed83a0\u003e: \n failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-4lmptk5vpx7s\" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\",\n \"name\": \"f4b98636-6475-41ec-a151-007f2ef479ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:44:01.234316083Z\",\n \"endTime\": \"2026-10-28T20:03:07.974415443Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56\n client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f06fbed83a0\u003e: \n failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-4lmptk5vpx7s\" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\",\n \"name\": \"f4b98636-6475-41ec-a151-007f2ef479ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:44:01.234316083Z\",\n \"endTime\": \"2026-10-28T20:03:07.974415443Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56\n client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f63bd757f8c437f8d2808c42bcc57ab70f3b2125a2a84565f38238af08e7d10","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try ag... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bb395824121b6b2126e9dcb99e19214721333dd8b9d9656e90089438ac8b9ddf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7816/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512185487626240","occurred_at":"2026-10-28T18:33:03Z","failed_at":"e2e","signature_id":"cce94bd2f5b67e5767a91df9d741230c8f8b592a2ad3598176e71b1aeada192a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-nxx9xr\"\nUnexpected error:\n \u003c*errors.joinError | 0x138a2d1f8048\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x138a2cdf5020\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x138a2cdf5000\u003e{\n msg: \"failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x138a2cdf4fe0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x138a2cd0dcc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-nxx9xr\"\nUnexpected error:\n \u003c*errors.joinError | 0x138a2d1f8048\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x138a2cdf5020\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x138a2cdf5000\u003e{\n msg: \"failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x138a2cdf4fe0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x138a2cd0dcc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0f63bd757f8c437f8d2808c42bcc57ab70f3b2125a2a84565f38238af08e7d10","d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7537/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509770164113408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7537,"pr_state":"closed","pr_sha":"c44baa7b0095a8353d51eb87fc63d1083a8bed12","final_merged_sha":"c44baa7b0095a8353d51eb87fc63d1083a8bed12","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T18:23:28Z","started_at":"2026-10-28T18:23:28Z","completed_at":"2026-10-28T20:43:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:52:20.691141632Z","timing_metadata_checked_at":"2026-10-28T20:52:20.691141632Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:52:20.69484442Z","region_metadata_checked_at":"2026-10-28T20:52:20.69484442Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509692493991936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7785,"pr_state":"open","pr_sha":"6fc7a87c7829c931d472bccf19660311bbe2e024","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T18:23:09Z","started_at":"2026-10-28T18:23:09Z","completed_at":"2026-10-28T20:50:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:03:20.825603054Z","timing_metadata_checked_at":"2026-10-28T21:03:20.825603054Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:03:20.84589959Z","region_metadata_checked_at":"2026-10-28T21:03:20.84589959Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a5d5bf92fe6bfee263dc51d0ed67000c057b3bc0a83f64141ee0e84d0cfc1a6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509692493991936","occurred_at":"2026-10-28T18:23:09Z","failed_at":"e2e","signature_id":"430a37f54fa88a8257498cad29293dfb1d7b7872cf2fdd8d9629d86a48de04bf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-kzwwft\"\nUnexpected error:\n \u003c*errors.joinError | 0x2fea822a5d0\u003e: \n VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record \"*.e2e-dns-czbqgt\" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n correlation ID: ef507981-8069-4371-b52a-e18fb184cf37\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-kzwwft\"\nUnexpected error:\n \u003c*errors.joinError | 0x2fea822a5d0\u003e: \n VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record \"*.e2e-dns-czbqgt\" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n correlation ID: ef507981-8069-4371-b52a-e18fb184cf37\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b703f4204991a61d0cf90f658410c32ca25e0cbc0d2e9af5a53fcc8458b106b4","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-czbqgt' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}},{"row_id":"f46ff22f2c8bf9986d649329f58257e7fd0f4fa6faa84118dd44eec258dce972","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509692493991936","occurred_at":"2026-10-28T18:23:09Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T19:42:52Z\n Ended: 2026-10-28T20:02:53Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3991936-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b703f4204991a61d0cf90f658410c32ca25e0cbc0d2e9af5a53fcc8458b106b4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509651545001984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7814,"pr_state":"open","pr_sha":"2f691cf38e8662d0ba7a42849efd5579c8f04de2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T18:22:59Z","started_at":"2026-10-28T18:22:59Z","completed_at":"2026-10-28T20:44:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:52:21.26325056Z","timing_metadata_checked_at":"2026-10-28T20:52:21.26325056Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:52:21.26628335Z","region_metadata_checked_at":"2026-10-28T20:52:21.26628335Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1d701df5bf88190cb2b7f47effc98418f5888486ef895e9c18b6f9a9911cb8d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509651545001984","occurred_at":"2026-10-28T18:22:59Z","failed_at":"e2e","signature_id":"d604f64973262ef8b5c3dcda3c973088263ebb3e1c8041dd41bfb3a67446230c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x67403d76ae0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0x67403d76ac0\u003e{\n observation: \u003c*errors.errorString | 0x674032428b0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0x67403d76aa0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x674036d06e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x67403d76a80\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x67404ab4ba0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x67403d76ae0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0x67403d76ac0\u003e{\n observation: \u003c*errors.errorString | 0x674032428b0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0x67403d76aa0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x674036d06e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x67403d76a80\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x67404ab4ba0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6249e15d0a6c7cf0142092c6b9eca0eb8f84826824bb608613ac26e30d3b14b0","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}},{"row_id":"2b606a3e4b8e2aa5dfcaafd98c4f602ca52e357f14e127281385f12e160f6eda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509651545001984","occurred_at":"2026-10-28T18:22:59Z","failed_at":"alert","signature_id":"980846bb94448eb2630686e4469bd6f85bb6d9608d00cf33c8152c9e5c035322","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T20:00:50Z\n Ended: 2026-10-28T20:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5001984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le\", pod=\"openshift-apiserver-7cb4d98fc-rzx96\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le/openshift-apiserver-7cb4d98fc-rzx96 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6249e15d0a6c7cf0142092c6b9eca0eb8f84826824bb608613ac26e30d3b14b0","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115506901079822336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"open","pr_sha":"e7c23babe29805031def86c975833c6d3badf2cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T18:12:04Z","started_at":"2026-10-28T18:12:04Z","completed_at":"2026-10-28T20:45:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:52:20.919310825Z","timing_metadata_checked_at":"2026-10-28T20:52:20.919310825Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:52:20.922859714Z","region_metadata_checked_at":"2026-10-28T20:52:20.922859714Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"24d67344a9d8f155f0339c719132ae18b5a2701b5c264838217f0b1b8705e479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115506901079822336","occurred_at":"2026-10-28T18:12:04Z","failed_at":"e2e","signature_id":"72129ad72c05e358b350979281ecdf024841a9958c65c7e23f6fa3993b142e2a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x31615201e9a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0x31615201e980\u003e{\n observation: \u003c*errors.errorString | 0x316153080da0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0x31615201e960\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x316150f60aa0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x31615201e920\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x316152c10a80\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x31615201e9a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0x31615201e980\u003e{\n observation: \u003c*errors.errorString | 0x316153080da0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0x31615201e960\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x316150f60aa0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x31615201e920\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x316152c10a80\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3102a1107f810d1b9d88041d585f2659521f004bd58b18f829fc59f73366743e","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}},{"row_id":"8716ce52ab4474d871799a4ada5822ecfc780f3b21ff531722f18870095a2e61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115506901079822336","occurred_at":"2026-10-28T18:12:04Z","failed_at":"e2e","signature_id":"4b1c26a0d10e93a5af6942386f37009deed677c3e114f68b9bdcf149db621ce3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ee0c1834d20\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster=\"np-version-upgrade-cluster-jl2sj8\" in resourcegroup=\"rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf\" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\",\n \"name\": \"267d3daf-f468-46eb-af3a-f1b72d573583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:42:27.649991944Z\",\n \"endTime\": \"2026-10-28T20:11:28.996573337Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23\n client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ee0c1834d20\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster=\"np-version-upgrade-cluster-jl2sj8\" in resourcegroup=\"rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf\" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\",\n \"name\": \"267d3daf-f468-46eb-af3a-f1b72d573583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:42:27.649991944Z\",\n \"endTime\": \"2026-10-28T20:11:28.996573337Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23\n client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","3102a1107f810d1b9d88041d585f2659521f004bd58b18f829fc59f73366743e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115500092713603072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"6405f41e9d626eac01c2304a4b0fb1eefd6c1e81","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T17:45:00Z","started_at":"2026-10-28T17:45:00Z","completed_at":"2026-10-28T20:16:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:21:08.711479293Z","timing_metadata_checked_at":"2026-10-28T20:21:08.711479293Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:21:08.718173172Z","region_metadata_checked_at":"2026-10-28T20:21:08.718173172Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3da1e83a52cba7aaf4809af885ebaed795d655ee97794ca7ced82d72d2df383f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115500092713603072","occurred_at":"2026-10-28T17:45:00Z","failed_at":"e2e","signature_id":"d552ce333ff64238cdb35341cbb435780b950120a489c5c39b9ea39ba2987f7f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-r5cltn\"\nUnexpected error:\n \u003c*errors.joinError | 0x263db2a8cd50\u003e: \n VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-p2v629\" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-r5cltn\"\nUnexpected error:\n \u003c*errors.joinError | 0x263db2a8cd50\u003e: \n VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-p2v629\" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7bc5b7efd9282638ed5ca12c2caee514f8621db6c432518ebabc0a7a96fa5799","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-p2v629' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7bc5b7efd9282638ed5ca12c2caee514f8621db6c432518ebabc0a7a96fa5799"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115491941603872768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"e2aa104a4154e9875791cd051472491971fa6720","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T17:12:37Z","started_at":"2026-10-28T17:12:37Z","completed_at":"2026-10-28T19:16:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:27:17.419005184Z","timing_metadata_checked_at":"2026-10-28T19:27:17.419005184Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T19:27:17.426324061Z","region_metadata_checked_at":"2026-10-28T19:27:17.426324061Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7811/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115488597250412544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7811,"pr_state":"open","pr_sha":"1c8be18c282105d7834cb0cc0132b9af1e06ec7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:59:20Z","started_at":"2026-10-28T16:59:20Z","completed_at":"2026-10-28T19:19:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:27:17.635185525Z","timing_metadata_checked_at":"2026-10-28T19:27:17.635185525Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T19:27:17.638772214Z","region_metadata_checked_at":"2026-10-28T19:27:17.638772214Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"02fbc84705d1692c71dd1048c6c516a758d093daabfbe56a040322e38a92cf4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7811/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115488597250412544","occurred_at":"2026-10-28T16:59:20Z","failed_at":"e2e","signature_id":"54271ff2159587172376438d500776a9f861603b592547af2dce7b7bf4770eaa","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc2b6fc92f40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4pl8j4mzdr2l\" to finish creating: GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n \"name\": \"67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T18:04:23.346337134Z\",\n \"endTime\": \"2026-10-28T18:23:31.470906003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-4pl8j4mzdr2l--managed/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-4pl8j4mzdr2l--managed/providers/Microsoft.Network/networkInterfaces/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f0916b6b-4fcf-4b8b-b7f8-5a66898b1190' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n correlation ID: 7d414729-3b90-4c4b-9628-b2993ebd631c\n client request ID: 0c6fdc27-946b-4c40-9ac6-c7629afa8ed5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc2b6fc92f40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4pl8j4mzdr2l\" to finish creating: GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n \"name\": \"67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T18:04:23.346337134Z\",\n \"endTime\": \"2026-10-28T18:23:31.470906003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-4pl8j4mzdr2l--managed/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-4pl8j4mzdr2l--managed/providers/Microsoft.Network/networkInterfaces/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f0916b6b-4fcf-4b8b-b7f8-5a66898b1190' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n correlation ID: 7d414729-3b90-4c4b-9628-b2993ebd631c\n client request ID: 0c6fdc27-946b-4c40-9ac6-c7629afa8ed5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7808/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115486053375676416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7808,"pr_state":"open","pr_sha":"027de3a2fb5aab6294565b7c33199bfac105900b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:49:13Z","started_at":"2026-10-28T16:49:13Z","completed_at":"2026-10-28T19:09:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:16:20.849151724Z","timing_metadata_checked_at":"2026-10-28T19:16:20.849151724Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T19:16:20.852654413Z","region_metadata_checked_at":"2026-10-28T19:16:20.852654413Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9860a8c6a4b8e2dea220e7c7a5afeb7ea5753f12920efa49a6b10863a8d74b78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7808/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115486053375676416","occurred_at":"2026-10-28T16:49:13Z","failed_at":"e2e","signature_id":"b7b55c0bb772493af2f0bf7b377817e6562c148d2c11c583c1c42088a1f89da3","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3756ccac8820\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n err: \u003c*errors.errorString | 0x3756cb62d010\u003e{\n s: \"dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3756ccac8820\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n err: \u003c*errors.errorString | 0x3756cb62d010\u003e{\n s: \"dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc8979b405de4c24ab4a1f29e1de7275444b774141612e5f89b0149929ee136c","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bc8979b405de4c24ab4a1f29e1de7275444b774141612e5f89b0149929ee136c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6951/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115485676949475328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6951,"pr_state":"open","pr_sha":"b79a3e336ee9c3a69ddd0279141eebeb7d687248","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-28T16:47:43Z","started_at":"2026-10-28T16:47:43Z","completed_at":"2026-10-28T19:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:06:13.546970032Z","timing_metadata_checked_at":"2026-10-28T19:06:13.546970032Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T19:06:13.550633021Z","region_metadata_checked_at":"2026-10-28T19:06:13.550633021Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478705303195648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7311,"pr_state":"open","pr_sha":"8131773635824fdc17eb2322bcb4ef5e1fc28734","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:20:01Z","started_at":"2026-10-28T16:20:01Z","completed_at":"2026-10-28T17:21:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T17:31:41.499318178Z","timing_metadata_checked_at":"2026-10-28T17:31:41.499318178Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T17:31:41.503235568Z","region_metadata_checked_at":"2026-10-28T17:31:41.503235568Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"6d6b4d051146a397008246d7f3faaa0e7980d0390a808dec0f20fcee364a4dec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478705303195648","occurred_at":"2026-10-28T16:20:01Z","failed_at":"alert","signature_id":"376cf6b8a607ae70f7c8bc27bdfad5a3d4307ff50aa806c754ca4f6361ee4e31","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T17:07:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-64b56768b9-r7v9d\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-64b56768b9-r7v9d has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-28T17:07:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"fleet\", pod=\"fleet-controller-7b9596cccb-g9thf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod fleet/fleet-controller-7b9596cccb-g9thf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-28T17:07:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-admin-api\", pod=\"admin-api-6d56468ffd-t9snj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-t9snj has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-28T17:07:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-admin-api\", pod=\"admin-api-6d56468ffd-rm24l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-rm24l has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"cb402748ac9ef1643e4bf444f9191ae6ff33ab223dc69287a865dc5839323e7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478705303195648","occurred_at":"2026-10-28T16:20:01Z","failed_at":"provision","signature_id":"b40184a1cf5ce039c231fc664b3fad303d9d873ec2b7da0a9a97597e803d23e4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T17:01:13.002Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-28T17:01:13.550Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-25 azureArmHelperIdentityClientId:ea5e9c6c-1933-464d-8bf6-db343c22b92d azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:ae6cf536-acc0-414c-ada8-f268aeeb97e3 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j3195648 cosmosDatabaseUrl:https://arohcpci01-rp-j3195648.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495] imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi monitoringApiGroup:monitoring.coreos.com ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj3195648.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj3195648.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:a5d5ff83-799c-44fd-a535-8c4386588d02 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-28T17:01:13.550Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:01:13.717Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-28T17:01:13.899Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-28T17:01:13.899Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:01:13.952Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:11:19.322Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded\"\ntime=2026-10-28T17:11:19.322Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8320dfc4cb472661bbb97b2880f78576cae510dd7ae52938db3db0330d5778","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","9a8320dfc4cb472661bbb97b2880f78576cae510dd7ae52938db3db0330d5778"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478468882862080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7693,"pr_state":"open","pr_sha":"0bb0b65e8a0937e1c8c0438e405c5860e7716e58","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:19:05Z","started_at":"2026-10-28T16:19:05Z","completed_at":"2026-10-28T16:55:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T17:00:19.652176858Z","timing_metadata_checked_at":"2026-10-28T17:00:19.652176858Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T17:00:19.660634333Z","region_metadata_checked_at":"2026-10-28T17:00:19.660634333Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ef1de41042a0575b8d14037768d0540c3d0cafcfa4045b01c12085bac32a4a5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478468882862080","occurred_at":"2026-10-28T16:19:05Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T16:43:28.511Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T16:43:31.273Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T16:43:32.448Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2862080-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87\ntime=2026-10-28T16:45:32.811Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/f309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87/operationStatuses/08584084010736719502\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/cluster-rybldmoz73lrm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2862080-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j2862080-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-28T16:47:32.817Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T16:47:34.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T16:47:35.068Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2862080-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006\ntime=2026-10-28T16:49:05.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006/operationStatuses/08584084008309075646\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7777,"pr_state":"open","pr_sha":"dc65ef0bc1e13ee1e6aaca5e8afa98d672663f78","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:16:33Z","started_at":"2026-10-28T16:16:33Z","completed_at":"2026-10-28T19:06:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T19:16:20.710396355Z","timing_metadata_checked_at":"2026-10-28T19:16:20.710396355Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T19:16:20.714108943Z","region_metadata_checked_at":"2026-10-28T19:16:20.714108943Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"24a58a5fc1e230461e2e7b03ef973cbb46ff8daef4a3a77caee5ff6abeb048a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","failed_at":"e2e","signature_id":"0de28270d5e084b0197def0b8c3e6a8fb86eded5a4eb6ea2c68a0150e9491a22","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2b12a0180120\u003e: \n resource group \"rg-candidate-4-22-9q8kddc4rfc6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hscqkp\" in resourcegroup=\"rg-candidate-4-22-9q8kddc4rfc6\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n \"name\": \"46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:51:06.474152018Z\",\n \"endTime\": \"2026-10-28T18:15:11.708953381Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d\n client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2b12a0180120\u003e: \n resource group \"rg-candidate-4-22-9q8kddc4rfc6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hscqkp\" in resourcegroup=\"rg-candidate-4-22-9q8kddc4rfc6\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n \"name\": \"46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:51:06.474152018Z\",\n \"endTime\": \"2026-10-28T18:15:11.708953381Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d\n client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"daac4af340e3c5c5f7b3b73acded2f5292a6260c6a7fd84241614003c956947f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aa9caeed1a5ff4322731ad0867507e3401f0ac9f69d75d7ddf799064553bcb0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","failed_at":"e2e","signature_id":"f42d951d643535f04588e61d133a18ff7cf5f9154020232391b99e2b14384551","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x17fd8fc22138\u003e: \n resource group \"kusto-logs-2tq8m2fllmvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2tq8m2fllmvs\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\",\n \"name\": \"e027677d-2084-4178-b70c-b4e22dfc37b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:46:42.22634078Z\",\n \"endTime\": \"2026-10-28T18:10:50.94970388Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756\n client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x17fd8fc22138\u003e: \n resource group \"kusto-logs-2tq8m2fllmvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2tq8m2fllmvs\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\",\n \"name\": \"e027677d-2084-4178-b70c-b4e22dfc37b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:46:42.22634078Z\",\n \"endTime\": \"2026-10-28T18:10:50.94970388Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756\n client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"daac4af340e3c5c5f7b3b73acded2f5292a6260c6a7fd84241614003c956947f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e4abb0a263715a97eb1a4c31d7a5bccbfc2961935a47923afc182696cc4c36e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","failed_at":"e2e","signature_id":"c0d8d5a449f24fff80c60b0a909a50844791a9e6857245b7bc3a62487b829544","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2a43d7c880a8\u003e: \n resource group \"private-kas-2t2bq848xclk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-2t2bq848xclk\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\",\n \"name\": \"827c464e-64e1-4e42-980f-59af842b0bed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:45:46.805727964Z\",\n \"endTime\": \"2026-10-28T18:09:51.562148289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399\n client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2a43d7c880a8\u003e: \n resource group \"private-kas-2t2bq848xclk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-2t2bq848xclk\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\",\n \"name\": \"827c464e-64e1-4e42-980f-59af842b0bed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:45:46.805727964Z\",\n \"endTime\": \"2026-10-28T18:09:51.562148289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399\n client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"daac4af340e3c5c5f7b3b73acded2f5292a6260c6a7fd84241614003c956947f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["daac4af340e3c5c5f7b3b73acded2f5292a6260c6a7fd84241614003c956947f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115476335995392000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7809,"pr_state":"open","pr_sha":"2a5af5f0f25085953117ef4c51755e96d1aa23bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T16:10:35Z","started_at":"2026-10-28T16:10:35Z","completed_at":"2026-10-28T18:17:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:24:25.698950443Z","timing_metadata_checked_at":"2026-10-28T18:24:25.698950443Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:24:25.701884334Z","region_metadata_checked_at":"2026-10-28T18:24:25.701884334Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e6f1ce37d4d42fccbed61fddba5907e4258c370240519333d169af3075f4a30d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115476335995392000","occurred_at":"2026-10-28T16:10:35Z","failed_at":"e2e","signature_id":"2cb0078a8e25d7023016829d59625bdf6e299ff4ac00dcfbbeb79a69cbfa76b3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c827a968340\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9k6kks4sqmhh\" to finish creating: GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n \"name\": \"e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:17:13.271950373Z\",\n \"endTime\": \"2026-10-28T17:36:19.733049446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n correlation ID: e5b7a290-af02-4ff7-9f0a-851a1b652878\n client request ID: a046022e-1fb3-43ea-9318-d95c8087e3da\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c827a968340\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9k6kks4sqmhh\" to finish creating: GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n \"name\": \"e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:17:13.271950373Z\",\n \"endTime\": \"2026-10-28T17:36:19.733049446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n correlation ID: e5b7a290-af02-4ff7-9f0a-851a1b652878\n client request ID: a046022e-1fb3-43ea-9318-d95c8087e3da\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115473563631751168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7786,"pr_state":"closed","pr_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","final_merged_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T15:59:12Z","started_at":"2026-10-28T15:59:12Z","completed_at":"2026-10-28T18:19:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:24:25.243364237Z","timing_metadata_checked_at":"2026-10-28T18:24:25.243364237Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:24:25.246533727Z","region_metadata_checked_at":"2026-10-28T18:24:25.246533727Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115472312617668608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7757,"pr_state":"open","pr_sha":"1e655faa8a9ae4e1edad0b4fb250400bf8ee7bb1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:54:37Z","started_at":"2026-10-28T15:54:37Z","completed_at":"2026-10-28T18:14:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:24:25.089551308Z","timing_metadata_checked_at":"2026-10-28T18:24:25.089551308Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:24:25.092970097Z","region_metadata_checked_at":"2026-10-28T18:24:25.092970097Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b1d10060c86694ba3fb41c99f629b50827489a6a9df810f44b5b313d2417e066","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115472312617668608","occurred_at":"2026-10-28T15:54:37Z","failed_at":"e2e","signature_id":"d8a52ca95428cb84af728b6065a7e821205b568d8191e5dba7ff7cf5f9250e85","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34a8b7c1a9e0\u003e: \n failed to create NodePool np-8844z2: failed waiting for nodepool=\"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\" in resourcegroup=\"rg-candidate-4-22-p8n8vcw2knj6\" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\",\n \"name\": \"e163bbab-c728-4e64-85d2-39934825ad2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:07:17.250607245Z\",\n \"endTime\": \"2026-10-28T17:26:25.565644744Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3\n client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34a8b7c1a9e0\u003e: \n failed to create NodePool np-8844z2: failed waiting for nodepool=\"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\" in resourcegroup=\"rg-candidate-4-22-p8n8vcw2knj6\" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\",\n \"name\": \"e163bbab-c728-4e64-85d2-39934825ad2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T17:07:17.250607245Z\",\n \"endTime\": \"2026-10-28T17:26:25.565644744Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3\n client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"822bfeba7eb34370c8d8683ec2656c86acd96391b5948409cfb9754f6ed523ca","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' s... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["822bfeba7eb34370c8d8683ec2656c86acd96391b5948409cfb9754f6ed523ca"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115467457228443648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7695,"pr_state":"open","pr_sha":"71709223d1ae0a86a0434791155d8fa1f1460000","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:35:19Z","started_at":"2026-10-28T15:35:19Z","completed_at":"2026-10-28T17:53:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:03:16.624560981Z","timing_metadata_checked_at":"2026-10-28T18:03:16.624560981Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:03:16.628197871Z","region_metadata_checked_at":"2026-10-28T18:03:16.628197871Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"91204419e5096ecff320bb58a6e5ad3d3ea875ed7596bdc966274d68535e4d9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115467457228443648","occurred_at":"2026-10-28T15:35:19Z","failed_at":"e2e","signature_id":"6ec2a0a084a83fcca9f0f2578989c9f523c37f120d4f7b1d92bf237362af37ee","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2db4fc89320\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zbnm2gmmrc7k\" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n \"name\": \"13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T16:52:57.688694522Z\",\n \"endTime\": \"2026-10-28T17:12:06.752450543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc\n client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2db4fc89320\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zbnm2gmmrc7k\" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n \"name\": \"13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T16:52:57.688694522Z\",\n \"endTime\": \"2026-10-28T17:12:06.752450543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc\n client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zbnm2gmmrc7k\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6951/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115465847387459584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6951,"pr_state":"open","pr_sha":"b79a3e336ee9c3a69ddd0279141eebeb7d687248","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:28:47Z","started_at":"2026-10-28T15:28:47Z","completed_at":"2026-10-28T15:40:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:45:32.980026828Z","timing_metadata_checked_at":"2026-10-28T15:45:32.980026828Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T15:45:32.986280509Z","region_metadata_checked_at":"2026-10-28T16:50:00.729470869Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"800640bed9dfd82aa3dbe61827e410a6d8bde7a0a2250746db5496eb1f8aea33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6951/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115465847387459584","occurred_at":"2026-10-28T15:28:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115464427141599232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7776,"pr_state":"closed","pr_sha":"bb1de194f6d1f4d1c23b575387e90e2c886c823b","final_merged_sha":"bb1de194f6d1f4d1c23b575387e90e2c886c823b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T15:23:17Z","started_at":"2026-10-28T15:23:17Z","completed_at":"2026-10-28T17:54:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:03:16.738421392Z","timing_metadata_checked_at":"2026-10-28T18:03:16.738421392Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:03:16.741960084Z","region_metadata_checked_at":"2026-10-28T18:03:16.741960084Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462359144206336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"open","pr_sha":"e7c23babe29805031def86c975833c6d3badf2cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:15:04Z","started_at":"2026-10-28T15:15:04Z","completed_at":"2026-10-28T17:40:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T17:52:47.14232502Z","timing_metadata_checked_at":"2026-10-28T17:52:47.14232502Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T17:52:47.14562431Z","region_metadata_checked_at":"2026-10-28T17:52:47.14562431Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"295e521a9c1f77d2ee968dd379d65affc4460ffc27f82c6f407111289b6d4650","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462359144206336","occurred_at":"2026-10-28T15:15:04Z","failed_at":"e2e","signature_id":"e95a85e5040a99543bdcfc0978c5a8dd4686775675fab4d33590a63d57268ab9","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x335ced2f1b40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfwkzrbn6vbm\" to finish creating: GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n \"name\": \"77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T16:28:12.935779531Z\",\n \"endTime\": \"2026-10-28T16:47:20.605052936Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-kfwkzrbn6vbm--managed/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfwkzrbn6vbm--managed/providers/Microsoft.Compute/virtualMachines/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n correlation ID: c83d59d4-8297-41d1-87c1-fbe0352a6163\n client request ID: 4dcc2ce6-6521-4d7f-b86f-f0309a80ecee\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x335ced2f1b40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfwkzrbn6vbm\" to finish creating: GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n \"name\": \"77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T16:28:12.935779531Z\",\n \"endTime\": \"2026-10-28T16:47:20.605052936Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-kfwkzrbn6vbm--managed/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfwkzrbn6vbm--managed/providers/Microsoft.Compute/virtualMachines/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n correlation ID: c83d59d4-8297-41d1-87c1-fbe0352a6163\n client request ID: 4dcc2ce6-6521-4d7f-b86f-f0309a80ecee\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462352114552832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7782,"pr_state":"closed","pr_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","final_merged_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T15:15:02Z","started_at":"2026-10-28T15:15:02Z","completed_at":"2026-10-28T17:46:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T17:52:47.277558408Z","timing_metadata_checked_at":"2026-10-28T17:52:47.277558408Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T17:52:47.2804707Z","region_metadata_checked_at":"2026-10-28T17:52:47.2804707Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462348259987456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"e2aa104a4154e9875791cd051472491971fa6720","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T15:15:01Z","started_at":"2026-10-28T15:15:01Z","completed_at":"2026-10-28T15:42:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:56:25.700996099Z","timing_metadata_checked_at":"2026-10-28T15:56:25.700996099Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T15:56:25.704335388Z","region_metadata_checked_at":"2026-10-28T17:00:19.767502105Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6f95064ca7154ee4eb8c099cf51a000a2b55fd200445d010acbd7aac3da4296e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462348259987456","occurred_at":"2026-10-28T15:15:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462331566657536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7802,"pr_state":"closed","pr_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","final_merged_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T15:14:57Z","started_at":"2026-10-28T15:14:57Z","completed_at":"2026-10-28T18:13:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T18:24:25.437260344Z","timing_metadata_checked_at":"2026-10-28T18:24:25.437260344Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T18:24:25.440752533Z","region_metadata_checked_at":"2026-10-28T18:24:25.440752533Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7a2b2725da7333bf380ee5f95505df98a7e5b99dcfaf8823ad9af02180c7a80f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462331566657536","occurred_at":"2026-10-28T15:14:57Z","failed_at":"e2e","signature_id":"4037be027d723586834ec2417ff728f3a460e286ca670e3bf3147bcf80036590","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-cptgtj\"\nUnexpected error:\n \u003c*errors.joinError | 0x199ff3994258\u003e: \n VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n [pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\n [pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\n [pod] condition type=PodReadyToStartContainers status=False reason= message=\n [pod] condition type=Initialized status=True reason= message=\n [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=PodScheduled status=True reason= message=\n [pod] container volume-check ready=false restartCount=0\n [pod] waiting reason=ContainerCreating message=\n [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\n [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-s4t9m/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\n {\n errs: [\n \u003c*fmt.wrapError | 0x199ff3d7a160\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n err: \u003c*fmt.wrapError | 0x199ff3d7a020\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n err: \u003c*verifiers.pollObservationError | 0x199ff3d7a680\u003e{\n observation: \u003c*errors.errorString | 0x199ff36faf30\u003e{\n s: \"pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n },\n interruption: \u003c*fmt.wrapError | 0x199ff3d7a660\u003e{\n msg: \"failed to get PVC e2e-csi-disk-s4t9m/e2e-csi-pvc: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-cptgtj\"\nUnexpected error:\n \u003c*errors.joinError | 0x199ff3994258\u003e: \n VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n [pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\n [pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\n [pod] condition type=PodReadyToStartContainers status=False reason= message=\n [pod] condition type=Initialized status=True reason= message=\n [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=PodScheduled status=True reason= message=\n [pod] container volume-check ready=false restartCount=0\n [pod] waiting reason=ContainerCreating message=\n [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\n [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-s4t9m/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\n [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\n {\n errs: [\n \u003c*fmt.wrapError | 0x199ff3d7a160\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n err: \u003c*fmt.wrapError | 0x199ff3d7a020\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n err: \u003c*verifiers.pollObservationError | 0x199ff3d7a680\u003e{\n observation: \u003c*errors.errorString | 0x199ff36faf30\u003e{\n s: \"pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n },\n interruption: \u003c*fmt.wrapError | 0x199ff3d7a660\u003e{\n msg: \"failed to get PVC e2e-csi-disk-s4t9m/e2e-csi-pvc: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c8baf4b2e1a6c519c2e6a72beff1cb671e4fa39d2c9d8669c830590e4eb74e17","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node \u003cnode\u003e (reason=ProvisioningFailed, count=8, age=7m53s)","search_query":"[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c8baf4b2e1a6c519c2e6a72beff1cb671e4fa39d2c9d8669c830590e4eb74e17"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462311480135680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7783,"pr_state":"closed","pr_sha":"a3c518a715e0119203bf4a48333d00445951072e","final_merged_sha":"a3c518a715e0119203bf4a48333d00445951072e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T15:14:49Z","started_at":"2026-10-28T15:14:49Z","completed_at":"2026-10-28T17:31:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T17:42:21.55177096Z","timing_metadata_checked_at":"2026-10-28T17:42:21.55177096Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T17:42:21.555459849Z","region_metadata_checked_at":"2026-10-28T17:42:21.555459849Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115460075651862528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7781,"pr_state":"open","pr_sha":"6f4a3de6266387917d1bf2132b8281794a51e08c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:05:59Z","started_at":"2026-10-28T15:05:59Z","completed_at":"2026-10-28T16:12:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T16:18:07.338474714Z","timing_metadata_checked_at":"2026-10-28T16:18:07.338474714Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T16:18:07.343648798Z","region_metadata_checked_at":"2026-10-28T16:18:07.343648798Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bd2bbcfb3cfae6171a8a323805a0e645105f7bd99f5dbf6bbbe1fd28ab565104","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115460075651862528","occurred_at":"2026-10-28T15:05:59Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T15:32:48.082Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T15:32:50.394Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T15:32:51.656Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1862528-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6\ntime=2026-10-28T15:35:22.185Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6/operationStatuses/08584084053144753487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/cluster-5xxgmkpcjnpl2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j1862528-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j1862528-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-28T15:37:22.187Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T15:37:23.185Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T15:37:24.435Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1862528-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f\ntime=2026-10-28T15:38:54.741Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/f330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f/operationStatuses/08584084050417029665\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115459200376115200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7780,"pr_state":"open","pr_sha":"d21a3fd6e10a70a717463fec163a4ef7e0afe41d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T15:02:31Z","started_at":"2026-10-28T15:02:31Z","completed_at":"2026-10-28T15:17:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:24:35.454248425Z","timing_metadata_checked_at":"2026-10-28T15:24:35.454248425Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T15:24:35.457167916Z","region_metadata_checked_at":"2026-10-28T16:28:20.413014719Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f0a7247fff62e57d64721b268dcfb9a60ca247b9a44f883744fc4ffe5f273414","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115459200376115200","occurred_at":"2026-10-28T15:02:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115458131596152832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7776,"pr_state":"closed","pr_sha":"6de3a5143f4ef47ba15ed6b443e8520641827d89","final_merged_sha":"bb1de194f6d1f4d1c23b575387e90e2c886c823b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T14:58:16Z","started_at":"2026-10-28T14:58:16Z","completed_at":"2026-10-28T15:00:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:24:35.353772228Z","timing_metadata_checked_at":"2026-10-28T15:24:35.353772228Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T15:24:35.357616416Z","region_metadata_checked_at":"2026-10-28T16:28:20.293957776Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d1e6e6009b4ba9498e4d0aa49f358ab0f14a79d474b7de1a5f5b667436be2d4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115458131596152832","occurred_at":"2026-10-28T14:58:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115457674907750400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7786,"pr_state":"closed","pr_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","final_merged_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T14:56:27Z","started_at":"2026-10-28T14:56:27Z","completed_at":"2026-10-28T15:21:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:24:35.634047182Z","timing_metadata_checked_at":"2026-10-28T15:24:35.634047182Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-28T15:24:35.63797647Z","region_metadata_checked_at":"2026-10-28T16:28:20.530633365Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"52a0e39391a93ace0b17b3282d39c88e2e0784a609cd7e85c43a2ab41a18a4bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115457674907750400","occurred_at":"2026-10-28T14:56:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7793,"pr_state":"open","pr_sha":"608bb71581814207d7cdc73f4914973721ae60a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T14:30:04Z","started_at":"2026-10-28T14:30:04Z","completed_at":"2026-10-28T14:55:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T15:03:32.590178734Z","timing_metadata_checked_at":"2026-10-28T15:03:32.590178734Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T15:03:32.594043923Z","region_metadata_checked_at":"2026-10-28T15:03:32.594043923Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"0d036531b9f313b98ef89e83cf76985134a5be45b34152b2015a1ded186df7ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"4e71503b0d08f969b9f6c986856e4d63c0ac55c20ae6ef6e411de2e3a1d59514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"5cedee8192cfb7579441fb120f6e9f1654c34a69a89c0654bd0ae28398b2a12f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","failed_at":"alert","signature_id":"d449b94908925109550a95da2d5df23c82ed852f6115ba04b76530f4e281f137","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115431844907323392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7783,"pr_state":"closed","pr_sha":"a3c518a715e0119203bf4a48333d00445951072e","final_merged_sha":"a3c518a715e0119203bf4a48333d00445951072e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T13:13:49Z","started_at":"2026-10-28T13:13:49Z","completed_at":"2026-10-28T14:03:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:11:58.919874919Z","timing_metadata_checked_at":"2026-10-28T14:11:58.919874919Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:11:58.92297901Z","region_metadata_checked_at":"2026-10-28T14:11:58.92297901Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9602632c2b0cf2db060dd38629ccba9e79e1cf1ccd7814bd15534575125d8809","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115431844907323392","occurred_at":"2026-10-28T13:13:49Z","failed_at":"provision","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T13:56:43.120Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n Kind: ARM\\n Template: templates/mgmt-nsp.bicep\\n Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-28T13:56:43.546Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-28T13:56:43.936Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7323392-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244\ntime=2026-10-28T13:58:14.248Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Resources/deployments/27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244/operationStatuses/08584084110817295270\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Resources/deployments/profile-sxizw5tprl6fg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7323392-mgmt-2/resourceAssociations/nsp-j7323392-mgmt-2-ra-4bf3dc7e-4183-5ff8-b0f7-92d18b1331a1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AssociationFailedAtResourceProvider\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Association provisioning failed at resource provider\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75e25b16c89c5bd443d1560c5ffb0c0789ec4d49846a07583c802aab771d1e1c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AssociationFailedAtResourceProvider; detail message Association provisioning failed at resource provider; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75e25b16c89c5bd443d1560c5ffb0c0789ec4d49846a07583c802aab771d1e1c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115431014208638976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7793,"pr_state":"open","pr_sha":"5a9d9b54f4ed4098943548d817bf169c7da3b98b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T13:10:09Z","started_at":"2026-10-28T13:10:09Z","completed_at":"2026-10-28T13:23:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T13:29:20.402779897Z","timing_metadata_checked_at":"2026-10-28T13:29:20.402779897Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T13:29:20.427419721Z","region_metadata_checked_at":"2026-10-28T14:33:07.869090561Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7d2ba676d50bbc415604616648a6fed7dce16479aa46615a44845990923732ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115431014208638976","occurred_at":"2026-10-28T13:10:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416954557698048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6718,"pr_state":"closed","pr_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","final_merged_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T12:14:38Z","started_at":"2026-10-28T12:14:38Z","completed_at":"2026-10-28T14:42:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:53:21.275562529Z","timing_metadata_checked_at":"2026-10-28T14:53:21.275562529Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:53:21.279119618Z","region_metadata_checked_at":"2026-10-28T14:53:21.279119618Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"157b5da823c657361e6e0659328023ad056498fe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T12:14:35Z","started_at":"2026-10-28T12:14:35Z","completed_at":"2026-10-28T14:27:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:33:07.76154519Z","timing_metadata_checked_at":"2026-10-28T14:33:07.76154519Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:33:07.76509618Z","region_metadata_checked_at":"2026-10-28T14:33:07.76509618Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"9c9ace4bfe3f3edc50e91e07ea3174caeb9e34ab5201f9f55b159b1bf6832eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","failed_at":"e2e","signature_id":"43128ad58b9794b202ee869cfaa57955ea19870ba55aba4a0092b4e1c16edd9c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xca1fe06c6d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xca1fe06c6d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d516b1df04e827a663b514ce5e826565130872a9cbbbc30e6eebf3f39c4bbbab","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0"}},{"row_id":"aa41b3fbc1bd443a8945b5a32907a4f9d0698bb632c9884ab38976bc91b99b81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","failed_at":"alert","signature_id":"5c1f0d2c18530e902522efe7242b86668fe04b921a7be0312c5fd330b98f0748","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T13:27:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.125:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-phdkm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5471b0ac-6a29-48cb-ab86-2893cd772fbd\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-phdkm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-28T13:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-84g47\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9462daaa-47d3-4b5e-b73b-01a9f857a137\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-84g47 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-28T13:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-84g47\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9462daaa-47d3-4b5e-b73b-01a9f857a137\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-84g47 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-28T13:32:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.125:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-phdkm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5471b0ac-6a29-48cb-ab86-2893cd772fbd\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-phdkm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"dea695286c3fcb0815993ca5c61a59bed6d8a96ed01b324d9b367d26b02162bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","failed_at":"e2e","signature_id":"636358cf2ad9136d60ba8fa1f4f46495eeb3f3f4d0f9eaa4658ff7f22e6bdd37","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc83633402b8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0xc83630bcee0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0xc8362e9fb78\u003e{\n errs: [\n \u003c*errors.errorString | 0xc83633b5610\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc83633402b8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0xc83630bcee0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0xc8362e9fb78\u003e{\n errs: [\n \u003c*errors.errorString | 0xc83633b5610\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","d516b1df04e827a663b514ce5e826565130872a9cbbbc30e6eebf3f39c4bbbab"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409868675354624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7802,"pr_state":"closed","pr_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","final_merged_sha":"c17975bf22f90eaf43e0c8ea61475cae7fbad58d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T11:46:29Z","started_at":"2026-10-28T11:46:29Z","completed_at":"2026-10-28T14:01:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:11:59.151151908Z","timing_metadata_checked_at":"2026-10-28T14:11:59.151151908Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:11:59.154014499Z","region_metadata_checked_at":"2026-10-28T14:11:59.154014499Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"defe2258b2e980008103dbe283ebba1773870364431f7f895b9466804014d8a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409868675354624","occurred_at":"2026-10-28T11:46:29Z","failed_at":"e2e","signature_id":"70032133cd194d044b701e3ebacbe2526af4f272899063e74ddee1c469f750e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-th8csc\"\nUnexpected error:\n \u003c*errors.joinError | 0x28585f5aa1e0\u003e: \n VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n [pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\n [pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\n [pod] condition type=PodReadyToStartContainers status=False reason= message=\n [pod] condition type=Initialized status=True reason= message=\n [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=PodScheduled status=True reason= message=\n [pod] container volume-check ready=false restartCount=0\n [pod] waiting reason=ContainerCreating message=\n [events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-5zpk7/e2e-csi-pvc\" (reason=Provisioning, count=10, age=36s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\n {\n errs: [\n \u003c*fmt.wrapError | 0x28585f462380\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n err: \u003c*fmt.wrapError | 0x28585e8a4800\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n err: \u003c*verifiers.pollObservationError | 0x28585e8a43c0\u003e{\n observation: \u003c*errors.errorString | 0x28585ee0a670\u003e{\n s: \"PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x28585e8a4320\u003e{\n msg: \"failed to get PVC e2e-csi-disk-5zpk7/e2e-csi-pvc: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-th8csc\"\nUnexpected error:\n \u003c*errors.joinError | 0x28585f5aa1e0\u003e: \n VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n [pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\n [pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\n [pod] condition type=PodReadyToStartContainers status=False reason= message=\n [pod] condition type=Initialized status=True reason= message=\n [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n [pod] condition type=PodScheduled status=True reason= message=\n [pod] container volume-check ready=false restartCount=0\n [pod] waiting reason=ContainerCreating message=\n [events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-5zpk7/e2e-csi-pvc\" (reason=Provisioning, count=10, age=36s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\n [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\n {\n errs: [\n \u003c*fmt.wrapError | 0x28585f462380\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n err: \u003c*fmt.wrapError | 0x28585e8a4800\u003e{\n msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod] waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n err: \u003c*verifiers.pollObservationError | 0x28585e8a43c0\u003e{\n observation: \u003c*errors.errorString | 0x28585ee0a670\u003e{\n s: \"PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x28585e8a4320\u003e{\n msg: \"failed to get PVC e2e-csi-disk-5zpk7/e2e-csi-pvc: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b616f5a6866cb92586bb6c324d97afd731ae9a87974504a140606af8f1c6d8c7","failure_pattern":"error code: CANCEL","search_query":"error code: CANCEL"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b616f5a6866cb92586bb6c324d97afd731ae9a87974504a140606af8f1c6d8c7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7782,"pr_state":"closed","pr_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","final_merged_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T11:45:41Z","started_at":"2026-10-28T11:45:41Z","completed_at":"2026-10-28T13:58:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:11:58.784117937Z","timing_metadata_checked_at":"2026-10-28T14:11:58.784117937Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:11:58.787699826Z","region_metadata_checked_at":"2026-10-28T14:11:58.787699826Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"205341b4e4f104a5e2fb10df538dbb30dd3576cdaddca336e2202c260239f5dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","failed_at":"alert","signature_id":"497a407fa6899d41490c9159b892ae9c0b2b92661b43d243f6d927eeba7fd4b9","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T13:22:31Z\n Ended: 2026-10-28T13:28:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/capi-provider is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired"}},{"row_id":"3bca439764d27623fe409ca77349ead7213ba2302cb56e70afea9e91413d638b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","failed_at":"alert","signature_id":"1c2cf09809fe509ac5ca6af7dd0c8a42eed9a467c897a7f2bb1d533ef52c7350","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T12:59:31Z\n Ended: 2026-10-28T13:28:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5496ebfd-2ab4-43b8-9f9a-e0483581f4bd\"\n Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-28T13:00:33Z\n Ended: 2026-10-28T13:27:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5496ebfd-2ab4-43b8-9f9a-e0483581f4bd\"\n Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"4e46c67345dca43f9e54fdbf3a21ec6bb05282849f37ba825fea5fae792ca257","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","failed_at":"e2e","signature_id":"fc014753f356fcc9708c367a6543a1d77e1f8177081d9e11992a0453189c6596","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-llxcnv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b596ab10140\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-llxcnv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-llxcnv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-468dv2nskdkq\" to finish creating: GET https://rp.j6945280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\",\n \"name\": \"e79580c0-76a4-4e28-820e-080632c7ada2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T12:38:01.23060152Z\",\n \"endTime\": \"2026-10-28T13:07:03.13964352Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6945280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\n correlation ID: 8869047d-1b08-4ff0-8212-00001a9147bd\n client request ID: 0bcb310d-278e-4828-86a5-41d1c5537311\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-llxcnv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b596ab10140\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-llxcnv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-llxcnv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-468dv2nskdkq\" to finish creating: GET https://rp.j6945280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\",\n \"name\": \"e79580c0-76a4-4e28-820e-080632c7ada2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T12:38:01.23060152Z\",\n \"endTime\": \"2026-10-28T13:07:03.13964352Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6945280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e79580c0-76a4-4e28-820e-080632c7ada2\n correlation ID: 8869047d-1b08-4ff0-8212-00001a9147bd\n client request ID: 0bcb310d-278e-4828-86a5-41d1c5537311\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"57e9b1b2bb99dc43f34f5937ac73714bfcacebb9a64721b561b8c0a1032219e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","failed_at":"alert","signature_id":"54df2b15da5ef9a1db45a3687ecb60f32af1e59282713521b839e992bda2ba7a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T13:12:32Z\n Ended: 2026-10-28T13:28:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ffee140f11b8fe452731c321fd8c5f24325826d7babdba17a061653d3f370f56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","failed_at":"alert","signature_id":"91827343ea1b42f538edbc36cb3bf02b91bb944d583eaf93fff5d26da344f32e","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T12:57:34Z\n Ended: 2026-10-28T13:23:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T12:57:34Z\n Ended: 2026-10-28T13:23:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7800/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115406309393174528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7800,"pr_state":"closed","pr_sha":"db216ea3f408ce9f198241729ae5f9ba4fa1a8fc","final_merged_sha":"db216ea3f408ce9f198241729ae5f9ba4fa1a8fc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T11:32:20Z","started_at":"2026-10-28T11:32:20Z","completed_at":"2026-10-28T13:43:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T13:51:19.579524589Z","timing_metadata_checked_at":"2026-10-28T13:51:19.579524589Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T13:51:19.582845078Z","region_metadata_checked_at":"2026-10-28T13:51:19.582845078Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7753/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115403809520881664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7753,"pr_state":"closed","pr_sha":"659a47e6b05209a86c11c1924a1b72a01a79d4a0","final_merged_sha":"659a47e6b05209a86c11c1924a1b72a01a79d4a0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T11:22:25Z","started_at":"2026-10-28T11:22:25Z","completed_at":"2026-10-28T13:57:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:01:38.072151234Z","timing_metadata_checked_at":"2026-10-28T14:01:38.072151234Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:01:38.075142725Z","region_metadata_checked_at":"2026-10-28T14:01:38.075142725Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7786,"pr_state":"closed","pr_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","final_merged_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T11:14:07Z","started_at":"2026-10-28T11:14:07Z","completed_at":"2026-10-28T13:57:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T14:01:38.214210903Z","timing_metadata_checked_at":"2026-10-28T14:01:38.214210903Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T14:01:38.217523893Z","region_metadata_checked_at":"2026-10-28T14:01:38.217523893Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"456f8edd6830febe8b3ed02a828293fc9e1d09d5264330edf986e576c851a313","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","failed_at":"e2e","signature_id":"20463c5543b2bf203bbfa488866984362e0c3b8eb919f697fb8074e808e6a4f1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-v9km2q\"\nUnexpected error:\n \u003c*errors.joinError | 0x1f131b624198\u003e: \n VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-swrng7\" not yet present in zone rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-swrng7' does not exist in resource group 'rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n correlation ID: f5ba2020-eed4-41d4-be4e-2d7ff36da168\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-v9km2q\"\nUnexpected error:\n \u003c*errors.joinError | 0x1f131b624198\u003e: \n VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-swrng7\" not yet present in zone rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-swrng7' does not exist in resource group 'rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n correlation ID: f5ba2020-eed4-41d4-be4e-2d7ff36da168\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a9db3befe442ad19ff8afe1f17b9e1c81ebea1992a9802344e12294deb314074","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-swrng7' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}},{"row_id":"a861e0029e052e2ae8a96ba179d654a8b41d1e0f2ea84dbfe550b300951ecd4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","failed_at":"e2e","signature_id":"a9ef1c5a61aa310aaaf853ef996be4e9f1f883da0ee59e7e69e9b3a3e5b3cd68","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-krfsmn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ab4bc30760\u003e: \n DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34ab4c426760\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-krfsmn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ab4bc30760\u003e: \n DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34ab4c426760\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"ecff3512f4039f20186a4302d2d1dd8db6f23c578715eefdbde67b3dc57f5fa0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","failed_at":"e2e","signature_id":"1ba167ed521beb385a26eb7e8a0f9de2df4f2d56b11f3ae2b5ab78e067c2563c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-x4d58d\"\nUnexpected error:\n \u003c*errors.joinError | 0x2a036e0a6030\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\": nodes \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2a036cc0d0a0\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2a036cc0d080\u003e{\n msg: \"failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2a036cc0d060\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2a036cc3c780\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-x4d58d\"\nUnexpected error:\n \u003c*errors.joinError | 0x2a036e0a6030\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\": nodes \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2a036cc0d0a0\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2a036cc0d080\u003e{\n msg: \"failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2a036cc0d060\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2a036cc3c780\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["a9db3befe442ad19ff8afe1f17b9e1c81ebea1992a9802344e12294deb314074","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401772536172544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"open","pr_sha":"e7c23babe29805031def86c975833c6d3badf2cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T11:13:52Z","started_at":"2026-10-28T11:13:52Z","completed_at":"2026-10-28T13:35:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T13:40:19.830617899Z","timing_metadata_checked_at":"2026-10-28T13:40:19.830617899Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T13:40:19.837970076Z","region_metadata_checked_at":"2026-10-28T13:40:19.837970076Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3149f6024714c6440e1b34c0e71877ac9fb2583d6062fb28b2f75b898be76609","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401772536172544","occurred_at":"2026-10-28T11:13:52Z","failed_at":"e2e","signature_id":"3d9c4de0a857dfc53c34c09e5f993fd7bb77e0bfa2613051219a1203b33d886f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd26a6364d80\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0xd26a6364d60\u003e{\n observation: \u003c*errors.errorString | 0xd26a57e61b0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0xd26a6364d40\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0xd26a5820dc0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0xd26a6364d20\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0xd26a719e570\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd26a6364d80\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n err: \u003c*verifiers.pollObservationError | 0xd26a6364d60\u003e{\n observation: \u003c*errors.errorString | 0xd26a57e61b0\u003e{\n s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n },\n interruption: \u003c*fmt.wrapError | 0xd26a6364d40\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0xd26a5820dc0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0xd26a6364d20\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0xd26a719e570\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"debe837728714debf1279c1ab16e7762e697cbb2740854aa543587193f51a693","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["debe837728714debf1279c1ab16e7762e697cbb2740854aa543587193f51a693"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401022980493312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7782,"pr_state":"closed","pr_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","final_merged_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T11:11:20Z","started_at":"2026-10-28T11:11:20Z","completed_at":"2026-10-28T11:14:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T11:21:04.359075725Z","timing_metadata_checked_at":"2026-10-28T11:21:04.359075725Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T11:21:04.362116216Z","region_metadata_checked_at":"2026-10-28T12:25:30.011906341Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"43cba374b524d040e410b4e95f76dfd145cbc1b32432998a130068b762485f8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401022980493312","occurred_at":"2026-10-28T11:11:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115394496836407296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7783,"pr_state":"closed","pr_sha":"a3c518a715e0119203bf4a48333d00445951072e","final_merged_sha":"a3c518a715e0119203bf4a48333d00445951072e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T10:45:24Z","started_at":"2026-10-28T10:45:24Z","completed_at":"2026-10-28T12:58:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T13:07:42.863320934Z","timing_metadata_checked_at":"2026-10-28T13:07:42.863320934Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T13:07:42.866682124Z","region_metadata_checked_at":"2026-10-28T13:07:42.866682124Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e0012085442948fc57a2b3ad6af9740deab53066dcaa6190e26b0b059957f51a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115394496836407296","occurred_at":"2026-10-28T10:45:24Z","failed_at":"e2e","signature_id":"7e41b58372eff808bb4c5baa7c863d948b46c4a604a47df1656a755e8ad17000","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x315531e198c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6tc6qf9c7brh\" to finish creating: GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n \"name\": \"4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T11:51:14.858832489Z\",\n \"endTime\": \"2026-10-28T12:10:21.145603579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u9f3h2h7j3o5r2t-np-1-frcss-9d6tp: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-6tc6qf9c7brh--managed/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6tc6qf9c7brh--managed/providers/Microsoft.Network/networkInterfaces/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n correlation ID: 80c82865-2a09-4afd-85b7-df3733e6ccd3\n client request ID: 14042924-6e78-44be-a159-f0ae94071a00\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x315531e198c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6tc6qf9c7brh\" to finish creating: GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n \"name\": \"4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T11:51:14.858832489Z\",\n \"endTime\": \"2026-10-28T12:10:21.145603579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u9f3h2h7j3o5r2t-np-1-frcss-9d6tp: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-6tc6qf9c7brh--managed/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6tc6qf9c7brh--managed/providers/Microsoft.Network/networkInterfaces/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n correlation ID: 80c82865-2a09-4afd-85b7-df3733e6ccd3\n client request ID: 14042924-6e78-44be-a159-f0ae94071a00\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115393597904785408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7795,"pr_state":"closed","pr_sha":"6fe05789052eeca044d6b70a365e3de4ace06ee7","final_merged_sha":"6fe05789052eeca044d6b70a365e3de4ace06ee7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T10:41:50Z","started_at":"2026-10-28T10:41:50Z","completed_at":"2026-10-28T13:06:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T13:18:32.704045389Z","timing_metadata_checked_at":"2026-10-28T13:18:32.704045389Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T13:18:32.711230067Z","region_metadata_checked_at":"2026-10-28T13:18:32.711230067Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376885591445504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"157b5da823c657361e6e0659328023ad056498fe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T09:35:25Z","started_at":"2026-10-28T09:35:25Z","completed_at":"2026-10-28T11:57:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T12:03:58.991110284Z","timing_metadata_checked_at":"2026-10-28T12:03:58.991110284Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T12:03:58.994316175Z","region_metadata_checked_at":"2026-10-28T12:03:58.994316175Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"458172cd10bcd534b0f7122341a1e39f0ffe5556acf8fd89423f2ae388574ce6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376885591445504","occurred_at":"2026-10-28T09:35:25Z","failed_at":"alert","signature_id":"f7f309c518710e27e7a8f712e71c207623d203053f761a3188198d64808a5e96","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T10:42:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.26:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-5dmcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6520e185-94fe-45c4-99a9-813a1698d907\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-5dmcb (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-28T10:43:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-ts4n7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a28f799-e0bd-4698-b612-8c6756c4b517\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-ts4n7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-28T10:43:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-ts4n7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a28f799-e0bd-4698-b612-8c6756c4b517\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-ts4n7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-28T10:47:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.26:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-5dmcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6520e185-94fe-45c4-99a9-813a1698d907\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-5dmcb (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"b4c8a1bc94030ff923ddb5ed1fa8986e54a1592c16147bdfe82dc72f721fc364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376885591445504","occurred_at":"2026-10-28T09:35:25Z","failed_at":"e2e","signature_id":"5ae5d9bea9a6a2cf4aba7447363670b8392610f5289f3a39616120110fd0bfc0","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2de4f1b0a5d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2de4f1b0a5d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4ce5bd36c087ccb995c35f95947eba01b62e03f0c2baf17243f6a3723c9f1559","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4ce5bd36c087ccb995c35f95947eba01b62e03f0c2baf17243f6a3723c9f1559","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376655219298304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6718,"pr_state":"closed","pr_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","final_merged_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T09:34:31Z","started_at":"2026-10-28T09:34:31Z","completed_at":"2026-10-28T11:48:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T11:53:44.524209172Z","timing_metadata_checked_at":"2026-10-28T11:53:44.524209172Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T11:53:44.527585162Z","region_metadata_checked_at":"2026-10-28T11:53:44.527585162Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2b85ce6b0c3c968348747f7c0aa28fe42a89ee329a8f7e84c4ec5ab25f922c60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376655219298304","occurred_at":"2026-10-28T09:34:31Z","failed_at":"e2e","signature_id":"da049643627fdd3d8b49937e7c216abea494324607b970c619b0669a8ec08194","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6750650e460\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x6750650e400\u003e{\n observation: \u003c*errors.errorString | 0x6750816aa40\u003e{\n s: \"dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x6750650e3e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x67506232820\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x6750650e3c0\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x67507892480\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6750650e460\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x6750650e400\u003e{\n observation: \u003c*errors.errorString | 0x6750816aa40\u003e{\n s: \"dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x6750650e3e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x67506232820\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x6750650e3c0\u003e{\n msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x67507892480\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2dc905970642341b5133d06f9629d7da5a13553cc2ccde3c27e0d2e405210927","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2dc905970642341b5133d06f9629d7da5a13553cc2ccde3c27e0d2e405210927"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115375119852376064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7775,"pr_state":"closed","pr_sha":"3b3384719c5a1520aba233f24370f02add587314","final_merged_sha":"3b3384719c5a1520aba233f24370f02add587314","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T09:28:22Z","started_at":"2026-10-28T09:28:22Z","completed_at":"2026-10-28T11:42:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T11:53:44.647288294Z","timing_metadata_checked_at":"2026-10-28T11:53:44.647288294Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T11:53:44.650197585Z","region_metadata_checked_at":"2026-10-28T11:53:44.650197585Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7797/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115374111151624192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7797,"pr_state":"closed","pr_sha":"558c3f455b9e74c8606664681a6704b6cccce61c","final_merged_sha":"558c3f455b9e74c8606664681a6704b6cccce61c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T09:24:24Z","started_at":"2026-10-28T09:24:24Z","completed_at":"2026-10-28T11:35:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T11:42:56.329588431Z","timing_metadata_checked_at":"2026-10-28T11:42:56.329588431Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T11:42:56.33304692Z","region_metadata_checked_at":"2026-10-28T11:42:56.33304692Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7706/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115372973631213568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7706,"pr_state":"closed","pr_sha":"a2f066d3e7d19fa7f34bfa5927a455748bf793d7","final_merged_sha":"a2f066d3e7d19fa7f34bfa5927a455748bf793d7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T09:19:53Z","started_at":"2026-10-28T09:19:53Z","completed_at":"2026-10-28T11:57:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T12:03:59.151671456Z","timing_metadata_checked_at":"2026-10-28T12:03:59.151671456Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T12:03:59.155030247Z","region_metadata_checked_at":"2026-10-28T12:03:59.155030247Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115364484842262528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"open","pr_sha":"e7c23babe29805031def86c975833c6d3badf2cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T08:46:09Z","started_at":"2026-10-28T08:46:09Z","completed_at":"2026-10-28T11:01:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T11:10:49.181331487Z","timing_metadata_checked_at":"2026-10-28T11:10:49.181331487Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T11:10:49.185215277Z","region_metadata_checked_at":"2026-10-28T11:10:49.185215277Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d65bf52c5eed1234675ea4c2185b1a9d6698ccc100d55d929bdf300caf3f95a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115364484842262528","occurred_at":"2026-10-28T08:46:09Z","failed_at":"e2e","signature_id":"78dd0668e8380bd8ed00795b6ff7617e72c8fc53789f3998be7247874f824a7b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b13ff3f4aa0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*errors.errorString | 0x2b13ffa1a410\u003e{\n s: \"dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b13ff3f4aa0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*errors.errorString | 0x2b13ffa1a410\u003e{\n s: \"dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eec089ccbb8d0a7f750b775b5e342570b85e76ed163473d22158ec1f98398c36","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["eec089ccbb8d0a7f750b775b5e342570b85e76ed163473d22158ec1f98398c36"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7798/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115349704135938048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7798,"pr_state":"open","pr_sha":"be7b63f9c5a1825aba62c0234ee1b36783cbbaf6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-28T07:47:25Z","started_at":"2026-10-28T07:47:25Z","completed_at":"2026-10-28T10:02:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T10:08:07.481458952Z","timing_metadata_checked_at":"2026-10-28T10:08:07.481458952Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T10:08:07.484276543Z","region_metadata_checked_at":"2026-10-28T10:08:07.484276543Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115349685118963712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7786,"pr_state":"closed","pr_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","final_merged_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T07:47:20Z","started_at":"2026-10-28T07:47:20Z","completed_at":"2026-10-28T10:02:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T10:08:07.323281525Z","timing_metadata_checked_at":"2026-10-28T10:08:07.323281525Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T10:08:07.328432609Z","region_metadata_checked_at":"2026-10-28T10:08:07.328432609Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb1158d748d1782595858285c965b0a0de6f48cc6a990a463b6624fee1f6b074","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115349685118963712","occurred_at":"2026-10-28T07:47:20Z","failed_at":"e2e","signature_id":"030be4640c04973776445b68ddb84b084e485fea34e2f96f0c190f92f6f2b41b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-tdqwb6\"\nUnexpected error:\n \u003c*errors.joinError | 0x122a895e0150\u003e: \n VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) timed out after 10m0.001s: DNS A record \"*.e2e-dns-lpggdd\" not yet present in zone rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-lpggdd' does not exist in resource group 'rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n correlation ID: fc55c026-0ddc-4f29-bb3d-f393fa0f3cd6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-tdqwb6\"\nUnexpected error:\n \u003c*errors.joinError | 0x122a895e0150\u003e: \n VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) timed out after 10m0.001s: DNS A record \"*.e2e-dns-lpggdd\" not yet present in zone rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-lpggdd' does not exist in resource group 'rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n correlation ID: fc55c026-0ddc-4f29-bb3d-f393fa0f3cd6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"19089bef7ce831b1ccaf302c1307f8a54f6b9033225b28b6d11d82d8f8c236be","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-lpggdd' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["19089bef7ce831b1ccaf302c1307f8a54f6b9033225b28b6d11d82d8f8c236be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7797/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115341484919099392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7797,"pr_state":"closed","pr_sha":"558c3f455b9e74c8606664681a6704b6cccce61c","final_merged_sha":"558c3f455b9e74c8606664681a6704b6cccce61c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T07:14:45Z","started_at":"2026-10-28T07:14:45Z","completed_at":"2026-10-28T09:18:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T09:27:13.774977246Z","timing_metadata_checked_at":"2026-10-28T09:27:13.774977246Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T09:27:13.778433236Z","region_metadata_checked_at":"2026-10-28T09:27:13.778433236Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fd734493c8af541d5b5c557b02307e167f483fe8aadb72dfe45e830d31e4ced6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7797/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115341484919099392","occurred_at":"2026-10-28T07:14:45Z","failed_at":"e2e","signature_id":"59c18568aacd2ba62746c2c266f8fd637816b7e9ca52d7fa2eca6c6ee1162774","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-5pj7h6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2590ea2143e0\u003e: \n DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2590e99482e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-5pj7h6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2590ea2143e0\u003e: \n DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2590e99482e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"157b5da823c657361e6e0659328023ad056498fe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T07:07:29Z","started_at":"2026-10-28T07:07:29Z","completed_at":"2026-10-28T09:13:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T09:16:35.874546823Z","timing_metadata_checked_at":"2026-10-28T09:16:35.874546823Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T09:16:35.877841915Z","region_metadata_checked_at":"2026-10-28T09:16:35.877841915Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"3a9b2df0e422fa37f4f0f6136d2de7bb0d361b50b879c2237ca8e3044a8e6253","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"e2e","signature_id":"ee364ce84dd90a160b7dc0e8bc2d5885b02b4ac45c05e55def20833e767b8add","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--95r4dzfzp4rt/cluster-hs-pre-5kmssk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc9a4e052980\u003e: \n failed to create HCP cluster cluster-hs-pre-5kmssk: failed waiting for cluster=\"cluster-hs-pre-5kmssk\" in resourcegroup=\"rg-hs-pre--95r4dzfzp4rt\" to finish creating: GET https://rp.j3004160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\",\n \"name\": \"8cb74587-080f-411c-9e53-c65b0814acf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T07:54:15.324506005Z\",\n \"endTime\": \"2026-10-28T08:23:21.521188973Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3004160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\n correlation ID: bc1ad563-f995-498c-a620-7b1f1cac6242\n client request ID: b82578c2-e97e-4b33-9a8e-529b76f26a9b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--95r4dzfzp4rt/cluster-hs-pre-5kmssk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc9a4e052980\u003e: \n failed to create HCP cluster cluster-hs-pre-5kmssk: failed waiting for cluster=\"cluster-hs-pre-5kmssk\" in resourcegroup=\"rg-hs-pre--95r4dzfzp4rt\" to finish creating: GET https://rp.j3004160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\",\n \"name\": \"8cb74587-080f-411c-9e53-c65b0814acf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T07:54:15.324506005Z\",\n \"endTime\": \"2026-10-28T08:23:21.521188973Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3004160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cb74587-080f-411c-9e53-c65b0814acf7\n correlation ID: bc1ad563-f995-498c-a620-7b1f1cac6242\n client request ID: b82578c2-e97e-4b33-9a8e-529b76f26a9b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"426a9216865555f9af3b9bc54cddb3fa8fcd044f591020e28b38a8030c14c58e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"e2e","signature_id":"2ed854a334795c3ea51fafdf41aa1c84cc3b8fdf14b1788a9d3550323b78a2db","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x20bd1e04e7a0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x20bd1e04e7a0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3921dee80f90c25a3f2a89623ede0bae1ddb59e58283210f34e1352a8c2ad0b2","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0"}},{"row_id":"9a426262aeea61d831849a5b3fbc683406902630c3d4b9b27c2b8b4607e6ccf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"alert","signature_id":"16049e988aa116aa463a1a9115799a6e2d39f159b5b564e13f1f9a9c31226285","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T08:17:22Z\n Ended: 2026-10-28T08:40:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"arohcp-monitor-kube-state-metrics-68ff96c687-7dfbl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T08:17:22Z\n Ended: 2026-10-28T08:40:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"arohcp-monitor-kube-state-metrics-68ff96c687-7dfbl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"9c7102887f8ea9ad8b1c394266b147a69b621af75ee831c917049f53dfc1a732","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"alert","signature_id":"dfd7afbd88dab1923cf763897f9f70a5cd1bd457893585b2411046095b00387c","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T08:13:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-krfd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2298911c-0832-493f-a9ef-f65728646e11\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-krfd7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-28T08:13:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-krfd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2298911c-0832-493f-a9ef-f65728646e11\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-krfd7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-28T08:15:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.208:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-fgmvz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e461328c-3664-4b8e-907f-f8f42456ba46\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-fgmvz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-28T08:15:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.208:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-fgmvz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e461328c-3664-4b8e-907f-f8f42456ba46\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-fgmvz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"ba74098d11c4fada68f1ed705e4aa84caaf1f1bc085b9b2b77347e99f510e676","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"alert","signature_id":"451b2e969d1680468440ceb90c398cbbb1804c59a23c31a9dd08adaa504f000a","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T08:20:17Z\n Ended: 2026-10-28T08:44:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a66d88b1-9c74-40e9-87a6-eb699288069f\"\n Description: Pod ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-28T08:20:17Z\n Ended: 2026-10-28T08:44:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a66d88b1-9c74-40e9-87a6-eb699288069f\"\n Description: Pod ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"f54747ba65ff07a4e2243c4bc20c2dd751dc621fb65f95b464c38d79c05a350c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","failed_at":"alert","signature_id":"d336eb4e6e6fc7c7bf769701f966b2fef1dc4f8dc73afdeb5d6c75eddd6e7a7a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-28T08:32:18Z\n Ended: 2026-10-28T08:41:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"arohcp-monitor-kube-state-metrics-68ff96c687-7dfbl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-28T08:32:18Z\n Ended: 2026-10-28T08:42:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/hcps-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s\", pod=\"arohcp-monitor-kube-state-metrics-68ff96c687-7dfbl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tnloambpnud3aq6p4sg9mdteg909a3n-z3x1x9z7m5g8p6s/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","3921dee80f90c25a3f2a89623ede0bae1ddb59e58283210f34e1352a8c2ad0b2","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339084032380928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7782,"pr_state":"closed","pr_sha":"f4e4b70a41f1523b4602b9b509befa6dbca579c1","final_merged_sha":"4ec109122cedf91bb07d3ae6b67ee559076e29ed","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-28T07:05:13Z","started_at":"2026-10-28T07:05:13Z","completed_at":"2026-10-28T09:22:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T09:27:13.377109635Z","timing_metadata_checked_at":"2026-10-28T09:27:13.377109635Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T09:27:13.380203326Z","region_metadata_checked_at":"2026-10-28T09:27:13.380203326Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7796/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115329871205896192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7796,"pr_state":"closed","pr_sha":"8ae9884b355ce85069adfdd36bc6d0f61eb07ca7","final_merged_sha":"8ae9884b355ce85069adfdd36bc6d0f61eb07ca7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T06:28:36Z","started_at":"2026-10-28T06:28:36Z","completed_at":"2026-10-28T08:42:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T08:54:59.651085863Z","timing_metadata_checked_at":"2026-10-28T08:54:59.651085863Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T08:54:59.654075754Z","region_metadata_checked_at":"2026-10-28T08:54:59.654075754Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115320649005142016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7795,"pr_state":"closed","pr_sha":"655931e34e3fb9e4fd5794e8d336d0771e58a4f2","final_merged_sha":"6fe05789052eeca044d6b70a365e3de4ace06ee7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T05:51:58Z","started_at":"2026-10-28T05:51:58Z","completed_at":"2026-10-28T08:18:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T08:33:14.160099907Z","timing_metadata_checked_at":"2026-10-28T08:33:14.160099907Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T08:33:14.164105594Z","region_metadata_checked_at":"2026-10-28T08:33:14.164105594Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f2adcf960e79a6e121e92817d7672fb4d597103b3b1e422ea15a139c1b90560a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115320649005142016","occurred_at":"2026-10-28T05:51:58Z","failed_at":"e2e","signature_id":"482b9f3179c10e36672118678ca7d0b64b36a887d80217ac5fb7f60bda8d3d24","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ecbca3c5be0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-p9l8lv6w4lzt\" to finish creating: GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n \"name\": \"0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T06:54:59.774467587Z\",\n \"endTime\": \"2026-10-28T07:14:00.6647054Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mxkn8-rr6bs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mxkn8-rr6bs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-mxkn8-c8z72: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-p9l8lv6w4lzt--managed/v20260901-np-1-mxkn8-c8z72-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-mxkn8-c8z72-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n correlation ID: 5492e0f1-946f-49bb-8474-ef7f078e1f6f\n client request ID: 12bebe6d-b861-48ec-942f-b913321fe64f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ecbca3c5be0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-p9l8lv6w4lzt\" to finish creating: GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n \"name\": \"0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T06:54:59.774467587Z\",\n \"endTime\": \"2026-10-28T07:14:00.6647054Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mxkn8-rr6bs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mxkn8-rr6bs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-mxkn8-c8z72: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-p9l8lv6w4lzt--managed/v20260901-np-1-mxkn8-c8z72-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-mxkn8-c8z72-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n correlation ID: 5492e0f1-946f-49bb-8474-ef7f078e1f6f\n client request ID: 12bebe6d-b861-48ec-942f-b913321fe64f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"fbfc1eab94893088b5d6f4b0d6d6224d411b5d0f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T03:31:21Z","started_at":"2026-10-28T03:31:21Z","completed_at":"2026-10-28T05:48:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T06:05:25.171516294Z","timing_metadata_checked_at":"2026-10-28T06:05:25.171516294Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T06:05:25.175523881Z","region_metadata_checked_at":"2026-10-28T06:05:25.175523881Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"6ccd8f08b9b43ce4eb158435b4f1b7d5e95783ea0f102e78294b99a01ddb8814","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"e2e","signature_id":"e8af4dcd98d7d05ebabf3df6907eac835155aa35d5063c6f3a85545c6af65f6c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x19a135b620a8\u003e: \n resource group \"feature-aggregation-c29rndpv7j5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c29rndpv7j5c\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\",\n \"name\": \"541f32be-fa69-4e38-80a2-84651b9746cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:58:24.120121666Z\",\n \"endTime\": \"2026-10-28T05:22:31.687389775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n correlation ID: efcef01e-78c0-4c18-a8a7-6bed6c2a5d41\n client request ID: 23dd3444-0c3c-438b-a21b-7fd27b67a689\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x19a135b620a8\u003e: \n resource group \"feature-aggregation-c29rndpv7j5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c29rndpv7j5c\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\",\n \"name\": \"541f32be-fa69-4e38-80a2-84651b9746cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:58:24.120121666Z\",\n \"endTime\": \"2026-10-28T05:22:31.687389775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n correlation ID: efcef01e-78c0-4c18-a8a7-6bed6c2a5d41\n client request ID: 23dd3444-0c3c-438b-a21b-7fd27b67a689\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"732dfaedbf5cfa705c3b2708a2147355d6a8d69ea19c05bc87f4cc8f64de8558","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"e2e","signature_id":"6ce95bc6ecd9c4fb5878916c5f1f43531cb7d0bd302c877f171028ac537ce75e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x867032a97d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x867032a97d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3cb444de9b5329e96be0f474504d91a498322635b3598692d70b1c8025a75f8b","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0"}},{"row_id":"986e40f8c32fae51e2d0a2be2813bb4400b9cf6abd8f37da834cf83cc896936c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"e2e","signature_id":"2bb63a1891dcda7670352e5cf272ec5241fe929f6c20cde1d439939704914208","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0xf5ad11741b0\u003e: \n resource group \"rg-candidate-5-0-lf2cjh6fwp8g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-28vlzn\" in resourcegroup=\"rg-candidate-5-0-lf2cjh6fwp8g\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n \"name\": \"ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:57:47.524111456Z\",\n \"endTime\": \"2026-10-28T05:21:51.687390361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n correlation ID: 57653937-379d-4e91-8cdc-72bb96f14f99\n client request ID: da17b50d-815e-4f7b-b6a1-511dd538a3b6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0xf5ad11741b0\u003e: \n resource group \"rg-candidate-5-0-lf2cjh6fwp8g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-28vlzn\" in resourcegroup=\"rg-candidate-5-0-lf2cjh6fwp8g\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n \"name\": \"ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:57:47.524111456Z\",\n \"endTime\": \"2026-10-28T05:21:51.687390361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n correlation ID: 57653937-379d-4e91-8cdc-72bb96f14f99\n client request ID: da17b50d-815e-4f7b-b6a1-511dd538a3b6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a509ea16cff3c7f75addd7d2c9af2e446eb567759a4f492197399ea89fa582b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"e2e","signature_id":"d21ae0d233481d01eee052f837a000b2cc41b93390d9f31d585d94518f7b055e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x19fdcb2b4318\u003e: \n resource group \"rg-candidate-4-22-b4lvcfvhwx52\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jsq2fr\" in resourcegroup=\"rg-candidate-4-22-b4lvcfvhwx52\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\",\n \"name\": \"6f392de4-c202-4b15-9624-974c650cb629\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:51:16.927309848Z\",\n \"endTime\": \"2026-10-28T05:15:21.671423449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n correlation ID: 6a7c51a3-41c5-4f0f-a544-ee94e68fe93c\n client request ID: cb15b9e0-1085-4828-b59c-20b42a78c8ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x19fdcb2b4318\u003e: \n resource group \"rg-candidate-4-22-b4lvcfvhwx52\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jsq2fr\" in resourcegroup=\"rg-candidate-4-22-b4lvcfvhwx52\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\",\n \"name\": \"6f392de4-c202-4b15-9624-974c650cb629\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:51:16.927309848Z\",\n \"endTime\": \"2026-10-28T05:15:21.671423449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n correlation ID: 6a7c51a3-41c5-4f0f-a544-ee94e68fe93c\n client request ID: cb15b9e0-1085-4828-b59c-20b42a78c8ff\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"afc0066f29f6bf4fa41bd5c09636d344fa97034304191d857bcbf9022efa6ca4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"alert","signature_id":"6717050d02899506dc19d396fc04fde0a13d0f143390e00656ec8a22fd39225c","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T04:40:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6816256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6816256/providers/microsoft.monitor/accounts/services-j6816256\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-d2g24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ff44de8f-6ae7-4cd3-9309-4206336bb4a3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-d2g24 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-28T04:42:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6816256-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.240:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6816256/providers/microsoft.monitor/accounts/services-j6816256\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-mw5bs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"911981bf-9bec-4f85-9bdd-db8c0cfd1db3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-mw5bs (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-28T04:43:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6816256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6816256/providers/microsoft.monitor/accounts/services-j6816256\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-d2g24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ff44de8f-6ae7-4cd3-9309-4206336bb4a3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-d2g24 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-28T04:45:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6816256-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.240:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6816256/providers/microsoft.monitor/accounts/services-j6816256\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-mw5bs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"911981bf-9bec-4f85-9bdd-db8c0cfd1db3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-mw5bs (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"c72a3e41d24164464a8f300f345a4503ab64c4e281a9662dccf38db3adc66cf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","failed_at":"e2e","signature_id":"2ea827d729fd1f9787e245343c8d8dc70b01e2bbe151064c5ee6e34c372663ce","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2fb9767520c0\u003e: \n resource group \"rg-candidate-4-23-8gsj9lgrgthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-x7cmb2\" in resourcegroup=\"rg-candidate-4-23-8gsj9lgrgthl\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n \"name\": \"c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:49:53.842986291Z\",\n \"endTime\": \"2026-10-28T05:14:02.592266097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n correlation ID: 4e7a26de-7852-4792-8175-683cd23ec9a8\n client request ID: a8483675-393f-439a-880f-cb8cebaae1a8\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n \u003c*errors.joinError | 0x2fb9767520c0\u003e: \n resource group \"rg-candidate-4-23-8gsj9lgrgthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-x7cmb2\" in resourcegroup=\"rg-candidate-4-23-8gsj9lgrgthl\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n \"name\": \"c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T04:49:53.842986291Z\",\n \"endTime\": \"2026-10-28T05:14:02.592266097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n correlation ID: 4e7a26de-7852-4792-8175-683cd23ec9a8\n client request ID: a8483675-393f-439a-880f-cb8cebaae1a8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","3cb444de9b5329e96be0f474504d91a498322635b3598692d70b1c8025a75f8b","40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7790/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115278850999980032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7790,"pr_state":"closed","pr_sha":"e1dbb294a6eb64d638ebb4f796017ac150939f7e","final_merged_sha":"e1dbb294a6eb64d638ebb4f796017ac150939f7e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T03:05:51Z","started_at":"2026-10-28T03:05:51Z","completed_at":"2026-10-28T05:10:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T05:23:58.007769967Z","timing_metadata_checked_at":"2026-10-28T05:23:58.007769967Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T05:23:58.011660355Z","region_metadata_checked_at":"2026-10-28T05:23:58.011660355Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115268754764992512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7791,"pr_state":"open","pr_sha":"fce1cc28fa2829da2f6702db24f3be8466ab8664","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T02:25:45Z","started_at":"2026-10-28T02:25:45Z","completed_at":"2026-10-28T05:13:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T05:23:58.277816355Z","timing_metadata_checked_at":"2026-10-28T05:23:58.277816355Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T05:23:58.280909646Z","region_metadata_checked_at":"2026-10-28T05:23:58.280909646Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"91d984e8b584138702aa7772ac384d31ecab4af2df3eff1d4b5c356deab046c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115268754764992512","occurred_at":"2026-10-28T02:25:45Z","failed_at":"e2e","signature_id":"1743e95f5bf74e9cfa2904f7bbd78e3557a8af048b321b3e791e03d323b382a0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-pfvhlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x5e040752480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hkoaxxduqas2k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Authorization/roleAssignments/1695ac58-5845-59e4-be60-94380c7b782a' at line '1' and column '21698'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 21698,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n correlation ID: c39b7be0-22bb-49c9-96d8-f53c5e4ad2fd\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-pfvhlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x5e040752480\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hkoaxxduqas2k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Authorization/roleAssignments/1695ac58-5845-59e4-be60-94380c7b782a' at line '1' and column '21698'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 21698,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n correlation ID: c39b7be0-22bb-49c9-96d8-f53c5e4ad2fd\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115268572052721664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"fbfc1eab94893088b5d6f4b0d6d6224d411b5d0f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T02:25:01Z","started_at":"2026-10-28T02:25:01Z","completed_at":"2026-10-28T03:19:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T03:39:54.564749409Z","timing_metadata_checked_at":"2026-10-28T03:39:54.564749409Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-28T03:39:54.568646997Z","region_metadata_checked_at":"2026-10-28T04:42:53.751747488Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68ee86668b068eaa986390fcf1911cc4b5bf311a4e2bf46c42acc74128b76be3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115268572052721664","occurred_at":"2026-10-28T02:25:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115261234742824960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7793,"pr_state":"open","pr_sha":"28823e2776561d5f56f15f6c776f0cbcfaaac2d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-28T01:55:52Z","started_at":"2026-10-28T01:55:52Z","completed_at":"2026-10-28T04:08:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:22:06.076526201Z","timing_metadata_checked_at":"2026-10-28T04:22:06.076526201Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:22:06.079231793Z","region_metadata_checked_at":"2026-10-28T04:22:06.079231793Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257822085124096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T01:42:18Z","started_at":"2026-10-28T01:42:18Z","completed_at":"2026-10-28T04:02:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:22:05.950940379Z","timing_metadata_checked_at":"2026-10-28T04:22:05.950940379Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:22:05.954136069Z","region_metadata_checked_at":"2026-10-28T04:22:05.954136069Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"fbfc1eab94893088b5d6f4b0d6d6224d411b5d0f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T01:41:17Z","started_at":"2026-10-28T01:41:17Z","completed_at":"2026-10-28T02:17:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T02:36:49.410521672Z","timing_metadata_checked_at":"2026-10-28T02:36:49.410521672Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T02:36:49.414239961Z","region_metadata_checked_at":"2026-10-28T02:36:49.414239961Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"23a7280ddb5d339540ad43f3a006d437b257f6d023a34b649fc54256c80862ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","failed_at":"alert","signature_id":"02713fa7ad9d7fa3c69367cb802a9482f205704d583c9d5a19a7d63dd9f94544","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2704640' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"8366e85a3e313a50caf0de0d8bf88253f92b9140ef92bc2982fd8366686a2013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2704640' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"c6d9c257acf9718fef5eb9d1db0a4dcb863fa0e2729739982948a1fa95cad8cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2704640' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115235750155915264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6081,"pr_state":"open","pr_sha":"860fe9f16ea44af6ed07ffbd4b0e9c062b01f3d7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-28T00:14:36Z","started_at":"2026-10-28T00:14:36Z","completed_at":"2026-10-28T02:29:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T02:47:00.394898774Z","timing_metadata_checked_at":"2026-10-28T02:47:00.394898774Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T02:47:00.400053658Z","region_metadata_checked_at":"2026-10-28T02:47:00.400053658Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"fccf89a235afbd8cebd31c000848ea09ba8f1cf73774b03f7c5f035f5676106b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115235750155915264","occurred_at":"2026-10-28T00:14:36Z","failed_at":"alert","signature_id":"dde57ce0fcd4d68140d4c1efab2ca64984da83e3c3b4ce209b60569f896e7311","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-28T01:28:46Z\n Severity: Sev3\n Labels: alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j5915264-svc\", severity=\"warning\"\n Description: Credential refresher expiration metric is missing for service cluster ci01-j5915264-svc.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}