{"meta":{"date":"2026-10-12","timezone":"UTC","generated_at":"2026-10-12T23:53:42Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":63,"failed_runs":50,"runs_with_occurrences":50,"runs_with_failure_pattern_matches":49,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":13,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"21c5097072b27775aeb5bfa1e803499de79c8013","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T21:12:49Z","started_at":"2026-10-12T21:12:49Z","completed_at":"2026-10-12T23:31:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T23:34:43.292623394Z","timing_metadata_checked_at":"2026-10-12T23:34:43.292623394Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T23:34:43.295697485Z","region_metadata_checked_at":"2026-10-12T23:34:43.295697485Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"345bc59fa9dfd71c380b093a9516e7df04a7467279e047e9e80fcdd61bb64b52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","failed_at":"e2e","signature_id":"e7d1ae872e2d1e1f299e0950e3c7121739a7ca3253cb34361fe8fe1de4b27d3a","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3990eaeace20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9d78f6chpq4q\" to finish creating: GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n \"name\": \"75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T22:04:42.193524873Z\",\n \"endTime\": \"2026-10-12T22:33:43.052616409Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n correlation ID: e27c7927-08d3-45ea-b9cc-9889d4e15592\n client request ID: e73a526e-7c8b-4130-ac0c-beedee83ba98\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3990eaeace20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9d78f6chpq4q\" to finish creating: GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n \"name\": \"75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T22:04:42.193524873Z\",\n \"endTime\": \"2026-10-12T22:33:43.052616409Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n correlation ID: e27c7927-08d3-45ea-b9cc-9889d4e15592\n client request ID: e73a526e-7c8b-4130-ac0c-beedee83ba98\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ef278caaad8de37337b1341295c8a824c6da90976f68a252ad9d8b029bbfae98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","failed_at":"e2e","signature_id":"8ec28f71b625b2c11ea506fd0748031d3bf5d074d3279546ef6bfc6cf0d20aaf","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754180636774400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"open","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T21:12:48Z","started_at":"2026-10-12T21:12:48Z","completed_at":"2026-10-12T23:21:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T23:24:27.942591179Z","timing_metadata_checked_at":"2026-10-12T23:24:27.942591179Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T23:24:27.945559069Z","region_metadata_checked_at":"2026-10-12T23:24:27.945559069Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d3383dd1a74009f806ff2ed6573b4e3c3ccb3ad0f00e5eca35b70fb79afb64f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754180636774400","occurred_at":"2026-10-12T21:12:48Z","failed_at":"alert","signature_id":"54a7b1283931df6f61be82ca63d92bf372c766203c87971d1d688d38deb1c668","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T23:02:04Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j6774400-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.46:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6774400/providers/microsoft.monitor/accounts/services-j6774400\", namespace=\"klusterlet-2tdgk5uq966q724hns0udujljsi1q0qu\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nxprp\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdgk5uq966q724hns0udujljsi1q0qu on cluster ci01-j6774400-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109750015583850496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7482,"pr_state":"open","pr_sha":"09888dd2e86e9dccf9108a5fdea62337320b1240","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T20:56:15Z","started_at":"2026-10-12T20:56:15Z","completed_at":"2026-10-12T23:18:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T23:24:28.039081566Z","timing_metadata_checked_at":"2026-10-12T23:24:28.039081566Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T23:24:28.041707957Z","region_metadata_checked_at":"2026-10-12T23:24:28.041707957Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109749870842613760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7428,"pr_state":"open","pr_sha":"b78ae90c1f1d34c65aa363e96ed18f47b3ac19a3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T20:55:41Z","started_at":"2026-10-12T20:55:41Z","completed_at":"2026-10-12T23:09:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T23:14:00.764634278Z","timing_metadata_checked_at":"2026-10-12T23:14:00.764634278Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T23:14:00.790576396Z","region_metadata_checked_at":"2026-10-12T23:14:00.790576396Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8d36c07040be2cf71e9d6858877b709b3b69a7ca987e975e67e9670f18d01488","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109749870842613760","occurred_at":"2026-10-12T20:55:41Z","failed_at":"e2e","signature_id":"69699f183ac245894ec15c2257ce86fb84bc1f5bed2cf85b38d4c739dedbd9d1","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"open","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T20:29:16Z","started_at":"2026-10-12T20:29:16Z","completed_at":"2026-10-12T22:47:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T23:03:07.645956581Z","timing_metadata_checked_at":"2026-10-12T23:03:07.645956581Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T23:03:07.64938797Z","region_metadata_checked_at":"2026-10-12T23:03:07.64938797Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"6b8eeff11645b9cb68cfc116644c17002740dabd2493fdcd8c6c4dcd04ab6c86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","failed_at":"alert","signature_id":"5cdd689b990f511feccbe071c3661b0d836cd6b71dcc489745470445ffd67711","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T22:07:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nlt8k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T22:07:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nlt8k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"b7dbe661a4037bcc5d326d8a145884a7f83404cf94426130982a1e5625b5c99d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","failed_at":"alert","signature_id":"a67e3b1ce6606aadaaf58795784360bda0ae262c0ccd8be147cd4aa4d8cfa336","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T22:24:20Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3618048-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.226:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/services-j3618048\", namespace=\"klusterlet-2tdg2drvs0bj4f4vu4h2o5drume4j1rc\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x9h2r\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdg2drvs0bj4f4vu4h2o5drume4j1rc on cluster ci01-j3618048-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"c25763aba14999cd6d69e383e11efbdc72f69f980e47fde96d9e7149ec81f64d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","failed_at":"alert","signature_id":"d4b2e29eb71d7ac5c954c156d7e1d4a7c4f83854fd89b2f4bf826b82158948d1","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T21:55:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T21:56:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T21:57:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-12T21:59:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109742847518314496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"open","pr_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T20:27:46Z","started_at":"2026-10-12T20:27:46Z","completed_at":"2026-10-12T22:37:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T22:41:55.006296191Z","timing_metadata_checked_at":"2026-10-12T22:41:55.006296191Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T22:41:55.009292281Z","region_metadata_checked_at":"2026-10-12T22:41:55.009292281Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"00dfa7a510d0d8c0e4e08fb45b84fbcf221f0ce4faa4eef7721ffc07530d4ba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109742847518314496","occurred_at":"2026-10-12T20:27:46Z","failed_at":"e2e","signature_id":"d4b16f343401a26bebbb19567d2aeea68fce56219f43f4bb17023f4763f88712","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109740763322519552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T20:19:29Z","started_at":"2026-10-12T20:19:29Z","completed_at":"2026-10-12T22:37:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T22:41:54.803993143Z","timing_metadata_checked_at":"2026-10-12T22:41:54.803993143Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T22:41:54.807398532Z","region_metadata_checked_at":"2026-10-12T22:41:54.807398532Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b3d90c069da49a58d8e2f75edea64020b76fe6ff121aa074ec9c717867f98c23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109740763322519552","occurred_at":"2026-10-12T20:19:29Z","failed_at":"alert","signature_id":"1f8e213972bab57fbe586fb0bcc701f04d1d69dc2413a26b541db4737b3a7545","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T22:06:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2519552-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2519552/providers/microsoft.monitor/accounts/hcps-j2519552\", namespace=\"ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nbzvl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T22:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2519552-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2519552/providers/microsoft.monitor/accounts/hcps-j2519552\", namespace=\"ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nbzvl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7472,"pr_state":"open","pr_sha":"d745401a7d4cf3ec4291d3ed03b2850c13b5fb63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T19:36:16Z","started_at":"2026-10-12T19:36:16Z","completed_at":"2026-10-12T22:24:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T22:31:42.445631559Z","timing_metadata_checked_at":"2026-10-12T22:31:42.445631559Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T22:31:42.448710249Z","region_metadata_checked_at":"2026-10-12T22:31:42.448710249Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"0f2a518ffa33c0a63cfeb2d25778a31e9a635e018df4e95cf61ad63dff58a0f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T20:59:27Z\n Ended: 2026-10-12T21:19:23Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6043392-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"68f154f473ba0f589783200dea5fe63b114c10d2a9864791ebc26f968dcb75b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","failed_at":"alert","signature_id":"0c18256aec4487d73b73645b8e1ded1390ad9218643d5179523cb223b9edd0b5","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T21:30:33Z\n Ended: 2026-10-12T22:07:32Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j6043392-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s\", region=\"westus3\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s (management cluster ci01-j6043392-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"de0541ef5fdfceec387e61071b366a1a97b32362ff824f8811e8d6f3d302e2b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","failed_at":"alert","signature_id":"4e6814427594ada40105d0c851f37c68ce6084d56587c92bb86a0a5911644b78","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T20:51:04Z\n Ended: 2026-10-12T20:57:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6043392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdfeve1hnh3ip3571ig9kpg8ogs5h0t-r4w0d6n8e5u2v7j\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdfeve1hnh3ip3571ig9kpg8ogs5h0t-r4w0d6n8e5u2v7j/etcd-2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T19:27:20Z","started_at":"2026-10-12T19:27:20Z","completed_at":"2026-10-12T19:50:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:04:40.90861836Z","timing_metadata_checked_at":"2026-10-12T20:04:40.90861836Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:04:40.912412548Z","region_metadata_checked_at":"2026-10-12T20:04:40.912412548Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"28395a6e67b56e476849fa77fba54bbff980520d828db7384fc6dbb3d2742a4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"46994566c791f752fadb4e695dfe2ef05eb3c96c8cff07cca62e1aa2f705c469","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","failed_at":"alert","signature_id":"3223dd11b35ca881a6905f2213555998ec56974987e667be15a6afbf19cd4f19","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"84ffe3f9ed70161ea4b8a7516819bdce0fab299ab77f9cac1926414832e16c1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109722197042925568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7481,"pr_state":"open","pr_sha":"22fe80c6bd390e9d619207de83cbe0eac2b03843","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T19:05:43Z","started_at":"2026-10-12T19:05:43Z","completed_at":"2026-10-12T21:30:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T21:39:52.59863318Z","timing_metadata_checked_at":"2026-10-12T21:39:52.59863318Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T21:39:52.601195773Z","region_metadata_checked_at":"2026-10-12T21:39:52.601195773Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7479,"pr_state":"closed","pr_sha":"0b9667bf3682c4991cb52e993a0e335801b3b039","final_merged_sha":"0b9667bf3682c4991cb52e993a0e335801b3b039","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T18:38:25Z","started_at":"2026-10-12T18:38:25Z","completed_at":"2026-10-12T20:52:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:57:46.465107031Z","timing_metadata_checked_at":"2026-10-12T20:57:46.465107031Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:57:46.467662622Z","region_metadata_checked_at":"2026-10-12T20:57:46.467662622Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"15a95693f7e3edbe2863585c4e84264fa876f7141bbc2c44a0e61f234907844b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","failed_at":"e2e","signature_id":"ba9594d183dcf679de486f3ef983cb99f7de22b5534227c654d1211082cc3ca0","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xe01e7de22c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x2v84z9v765\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\",\n \"name\": \"3d382676-ff60-4e1f-babd-b92045c21565\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:22:34.907448575Z\",\n \"endTime\": \"2026-10-12T19:51:42.771361832Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n correlation ID: 7441ce1a-f7e5-42c1-bf5b-99ce537aeb48\n client request ID: fed3af2b-11b1-43b5-8577-26857bd2a8c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xe01e7de22c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x2v84z9v765\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\",\n \"name\": \"3d382676-ff60-4e1f-babd-b92045c21565\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:22:34.907448575Z\",\n \"endTime\": \"2026-10-12T19:51:42.771361832Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n correlation ID: 7441ce1a-f7e5-42c1-bf5b-99ce537aeb48\n client request ID: fed3af2b-11b1-43b5-8577-26857bd2a8c1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5345ddfdb9e606e08e307ee504be7a3eae2eb6d561e21d88d7c4e5b176d001ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","failed_at":"alert","signature_id":"a25524a009142bcd67f70022a288b9e9be2947605657927be7457aeb02bcc1d2","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:45:53Z\n Ended: 2026-10-12T20:13:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9ddee187-d906-4221-b48f-73e1102c7b1e\"\n Description: Pod ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-12T19:45:53Z\n Ended: 2026-10-12T20:13:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9ddee187-d906-4221-b48f-73e1102c7b1e\"\n Description: Pod ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"7714be08e1efcd3c4272b31041ad1a123f09034476db0294adaa8b1d4891ea72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","failed_at":"alert","signature_id":"01b14f16705c690af3cc17499286b7047bac4716208b40b6f4d834caa2ce6ebc","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:53:49Z\n Ended: 2026-10-12T20:09:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cdb52\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T19:53:49Z\n Ended: 2026-10-12T20:13:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cdb52\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"be05f5ab8de6e70b218b7d94aed92325e1b2aad688618e1dd87a69041f7aff1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","failed_at":"e2e","signature_id":"0b8a3b5437beafe0105aa62d86153fdc67ea6aaf34efdb9197853845e0812245","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109709852522582016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"open","pr_sha":"abae04ffad0c9e129bcefcc66072d5b64e0aa705","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T18:16:39Z","started_at":"2026-10-12T18:16:39Z","completed_at":"2026-10-12T20:44:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:57:45.541589113Z","timing_metadata_checked_at":"2026-10-12T20:57:45.541589113Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:57:45.544656003Z","region_metadata_checked_at":"2026-10-12T20:57:45.544656003Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1dc485c78b117ba2d9315eb91b3e3075563a416cc1c978a8c874ee2cee9273dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109709852522582016","occurred_at":"2026-10-12T18:16:39Z","failed_at":"e2e","signature_id":"f07f78c84cfe8e58c2511543b04e0ca7a20c652e502f4d4a311097ea89b1549b","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d62ef202100\u003e: \n failed to create NodePool np-d4z57m: failed waiting for nodepool=\"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\" in resourcegroup=\"rg-hs-pre--zplpn9jd4vzj\" to finish creating: GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\",\n \"name\": \"2867897e-0970-460c-81d6-4c5fa1ec8583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:39:38.582635827Z\",\n \"endTime\": \"2026-10-12T20:08:43.787852166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n correlation ID: 56d0624b-f6fa-49bb-b96e-3fa32af8a523\n client request ID: c9d4e044-239c-4fd2-8127-3707455ec72b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d62ef202100\u003e: \n failed to create NodePool np-d4z57m: failed waiting for nodepool=\"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\" in resourcegroup=\"rg-hs-pre--zplpn9jd4vzj\" to finish creating: GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\",\n \"name\": \"2867897e-0970-460c-81d6-4c5fa1ec8583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:39:38.582635827Z\",\n \"endTime\": \"2026-10-12T20:08:43.787852166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n correlation ID: 56d0624b-f6fa-49bb-b96e-3fa32af8a523\n client request ID: c9d4e044-239c-4fd2-8127-3707455ec72b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109708565542342656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"open","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T18:11:33Z","started_at":"2026-10-12T18:11:33Z","completed_at":"2026-10-12T20:43:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:57:45.759627662Z","timing_metadata_checked_at":"2026-10-12T20:57:45.759627662Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:57:45.762540552Z","region_metadata_checked_at":"2026-10-12T20:57:45.762540552Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9068c8f91731dac083218dce43a2da58f39aae09ae03f94e8b60d8df4cf2ce7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109708565542342656","occurred_at":"2026-10-12T18:11:33Z","failed_at":"e2e","signature_id":"8d1213d4562f39353e445c7a58b0fd1c57fd1080920c955cf4d994b8f4a3de3c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109706138382503936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7470,"pr_state":"open","pr_sha":"5f945eb6dd49456a91e7a07766aaa8e77ac88803","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T18:01:54Z","started_at":"2026-10-12T18:01:54Z","completed_at":"2026-10-12T20:43:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:57:46.243222896Z","timing_metadata_checked_at":"2026-10-12T20:57:46.243222896Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:57:46.24788398Z","region_metadata_checked_at":"2026-10-12T20:57:46.24788398Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d295211212317bbd76ccb5eb841335e9bd02eb404c058f935a9a50609cabaad4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109706138382503936","occurred_at":"2026-10-12T18:01:54Z","failed_at":"alert","signature_id":"1e41fec641cef36701893933a54d28e9a3e41519280905104056febcd0e7dcfb","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:50:18Z\n Ended: 2026-10-12T20:25:22Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2503936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l\", region=\"centralus\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l (management cluster ci01-j2503936-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109705158475649024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7471,"pr_state":"open","pr_sha":"e07c83dc0f832925b6a2ff2ede9de2c885f300c7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T17:58:00Z","started_at":"2026-10-12T17:58:00Z","completed_at":"2026-10-12T20:28:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:36:14.669333256Z","timing_metadata_checked_at":"2026-10-12T20:36:14.669333256Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:36:14.672132548Z","region_metadata_checked_at":"2026-10-12T20:36:14.672132548Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e0931705c5b51ce6ec1cd1772ee23f76fd4d79b68c501a3eedeafaa24af3fecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109705158475649024","occurred_at":"2026-10-12T17:58:00Z","failed_at":"alert","signature_id":"1de0d91a762f7a24003e4b3846ddd93be7eb53f9d5a9baa369ea319d275302ba","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:45:35Z\n Ended: 2026-10-12T20:14:37Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j5649024-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z\", region=\"westus3\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z (management cluster ci01-j5649024-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7467,"pr_state":"open","pr_sha":"395ae2305c3515cbbbe03da6d1c4c686fc44b301","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T17:55:03Z","started_at":"2026-10-12T17:55:03Z","completed_at":"2026-10-12T20:21:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:36:14.214828517Z","timing_metadata_checked_at":"2026-10-12T20:36:14.214828517Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:36:14.217520909Z","region_metadata_checked_at":"2026-10-12T20:36:14.217520909Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"32fbca9c390c1b71b4565c294dea0a25b28b584f7222cad66d2b7c160d26030f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","failed_at":"e2e","signature_id":"01033c5c618998eef4c756c29b116f1c2e1c20ad91b1f1d16200cebb174aafe7","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd4464611460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd4464611460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"531a2084e8686f9a42755ac056f2dd31faa67fc9e5c5979d05857ee0dfb1b975","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c923bf9aa2e5ba0b8d510490b39ac95158baeea6173d4d7b5364cf0a75615484","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","failed_at":"e2e","signature_id":"317dedee167ea472d7b8eb2f6ca44fba55d08af18e00d84e205137d4efe864b6","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d22bb436fc039bdee0e499ca5aac864761984cd1d47c70d2b2e4d699c30dfc96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","failed_at":"e2e","signature_id":"01fffc7b6758239c11fdf9bc2bb6322f5a22ddef4368d43104f752d3e34ce9e2","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b89623a31d2776290135b67a1c3e8137de86d30ee19f034e5531fb48ad00d00f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["531a2084e8686f9a42755ac056f2dd31faa67fc9e5c5979d05857ee0dfb1b975","7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","b89623a31d2776290135b67a1c3e8137de86d30ee19f034e5531fb48ad00d00f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109699843009024000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7468,"pr_state":"open","pr_sha":"209c55bcf8a51457cd4564e430b8501d7e8956fa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T17:36:53Z","started_at":"2026-10-12T17:36:53Z","completed_at":"2026-10-12T20:18:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:36:14.487460901Z","timing_metadata_checked_at":"2026-10-12T20:36:14.487460901Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:36:14.489915394Z","region_metadata_checked_at":"2026-10-12T20:36:14.489915394Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109681427179114496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7376,"pr_state":"open","pr_sha":"521d3fc4b21cde036d1510f6fdc6897c0b831735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T16:23:42Z","started_at":"2026-10-12T16:23:42Z","completed_at":"2026-10-12T18:33:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T18:41:39.488942701Z","timing_metadata_checked_at":"2026-10-12T18:41:39.488942701Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T18:41:39.492038692Z","region_metadata_checked_at":"2026-10-12T18:41:39.492038692Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7478/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109680616554369024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7478,"pr_state":"closed","pr_sha":"5cffec4f6fb6314bc6a88e2707d136a756eb4a03","final_merged_sha":"5cffec4f6fb6314bc6a88e2707d136a756eb4a03","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T16:20:29Z","started_at":"2026-10-12T16:20:29Z","completed_at":"2026-10-12T18:35:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T18:41:39.888922285Z","timing_metadata_checked_at":"2026-10-12T18:41:39.888922285Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T18:41:39.891461578Z","region_metadata_checked_at":"2026-10-12T18:41:39.891461578Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"21c5097072b27775aeb5bfa1e803499de79c8013","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T16:01:28Z","started_at":"2026-10-12T16:01:28Z","completed_at":"2026-10-12T18:33:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T18:41:39.215831831Z","timing_metadata_checked_at":"2026-10-12T18:41:39.215831831Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T18:41:39.219149221Z","region_metadata_checked_at":"2026-10-12T18:41:39.219149221Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9068402f5e7bb5f25bc6012abf5658afe16fb469d72d8323b115a73ffe3bf54f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","failed_at":"e2e","signature_id":"22c47845096418cb71e4fa35973233fa4b21f339d382954b3aee25b82c29cb84","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-zx5s4v\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-zx5s4v\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36"}},{"row_id":"aa00a7c4f948914d75ef9584b19239738b470b89e6e92536efd0e24ea093fffc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","failed_at":"e2e","signature_id":"a87f271596a6f4e182a266365db74092bd67d3130b10c9e295b2604bd480ee54","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df811d0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df140c0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019e2caff0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df811d0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df140c0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019e2caff0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0a9bcf49d179171c63dccafd8dbb4e653af27553fa78df236def98914f6ada90","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0a9bcf49d179171c63dccafd8dbb4e653af27553fa78df236def98914f6ada90","bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"open","pr_sha":"f71a5d1f427dd2f2dbcafb2df4f4502fd8be8450","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T15:17:08Z","started_at":"2026-10-12T15:17:08Z","completed_at":"2026-10-12T17:36:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:37:59.077431755Z","timing_metadata_checked_at":"2026-10-12T17:37:59.077431755Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:37:59.079868947Z","region_metadata_checked_at":"2026-10-12T17:37:59.079868947Z"},"failed_at":["alert","e2e"],"failed_test_count":53,"occurrences":[{"row_id":"06983bcd414f72cd1b5686ab4c794b8fffeafc6986fe1b134194497419deabec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"9b8bdf9e25347ed8bbd86e38e47f8af0335efec4efa7ea5222d2c6561d3fadfb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-zf2xkx\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x20d83d0b6300\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-zf2xkx: failed waiting for cluster=\"cp-ystream-upgrade-4-22-zf2xkx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-z4gr6cw446tn\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\",\n \"name\": \"4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:51.296200468Z\",\n \"endTime\": \"2026-10-12T16:33:59.904433857Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\n correlation ID: 533c542a-908f-40a3-ad31-456c1c10be02\n client request ID: b534539f-cc46-4687-aa71-9ff6931d4410\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-zf2xkx\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x20d83d0b6300\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-zf2xkx: failed waiting for cluster=\"cp-ystream-upgrade-4-22-zf2xkx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-z4gr6cw446tn\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\",\n \"name\": \"4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:51.296200468Z\",\n \"endTime\": \"2026-10-12T16:33:59.904433857Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a0ccc8b-6d58-40e9-9350-d1b7cc14d624\n correlation ID: 533c542a-908f-40a3-ad31-456c1c10be02\n client request ID: b534539f-cc46-4687-aa71-9ff6931d4410\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"08024ef74dfe16d947478d8443c15120fc10fd44de90b1ed02046ec8ce8f9013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"6b095616bded2736b44fd15719c1d4b68c8ddb854dc8491ee2f30e52888a0be7","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x15d41e5c6240\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j4jfht4qlmg\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\",\n \"name\": \"2da82feb-2183-4a70-9d57-b0fc9cc5fb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:19.765893273Z\",\n \"endTime\": \"2026-10-12T16:34:19.389105786Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\n correlation ID: 9be86eb4-1760-485e-aa02-0152aaa7b2dd\n client request ID: ea7e40c3-adce-40f3-9337-36426044a94c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x15d41e5c6240\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j4jfht4qlmg\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\",\n \"name\": \"2da82feb-2183-4a70-9d57-b0fc9cc5fb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:19.765893273Z\",\n \"endTime\": \"2026-10-12T16:34:19.389105786Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2da82feb-2183-4a70-9d57-b0fc9cc5fb71\n correlation ID: 9be86eb4-1760-485e-aa02-0152aaa7b2dd\n client request ID: ea7e40c3-adce-40f3-9337-36426044a94c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0d25405ac6cacc1b380aa94f0ef26be847eeeecfff66fc2b0ac683ffe0b35798","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"d859241facd880cf6291230bc4ecdb749dce2456ddc8c4a192721ae181952d05","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x207341b0e2c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xtxmlffltb9n\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\",\n \"name\": \"e92df330-b202-4f70-bed1-e815e94ef4f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:14.427882995Z\",\n \"endTime\": \"2026-10-12T16:34:19.650879023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\n correlation ID: b072c371-50d5-4b18-b7e4-f792c3250769\n client request ID: 861c25eb-91cb-4175-813e-baf5e45adfb4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x207341b0e2c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xtxmlffltb9n\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\",\n \"name\": \"e92df330-b202-4f70-bed1-e815e94ef4f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:14.427882995Z\",\n \"endTime\": \"2026-10-12T16:34:19.650879023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e92df330-b202-4f70-bed1-e815e94ef4f3\n correlation ID: b072c371-50d5-4b18-b7e4-f792c3250769\n client request ID: 861c25eb-91cb-4175-813e-baf5e45adfb4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"20f64056da5ff7efa48ae06f867dc300100a5273af0f9d7ca5c9455e63ee1bd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"5bc4f6a77655afb07a6100dcfe611731a929b63429a3f3272baa0c1801222753","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x1042c14a00e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nr5sf886lw6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\",\n \"name\": \"6dac1a74-819f-41d7-8d23-79db0084d547\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:51.742280808Z\",\n \"endTime\": \"2026-10-12T16:33:59.639369344Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\n correlation ID: d7935d7f-d7f9-4e3e-9404-6468dba0cf73\n client request ID: 448194cd-f1a7-4755-85d1-3802ea5135b8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x1042c14a00e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nr5sf886lw6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\",\n \"name\": \"6dac1a74-819f-41d7-8d23-79db0084d547\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:51.742280808Z\",\n \"endTime\": \"2026-10-12T16:33:59.639369344Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dac1a74-819f-41d7-8d23-79db0084d547\n correlation ID: d7935d7f-d7f9-4e3e-9404-6468dba0cf73\n client request ID: 448194cd-f1a7-4755-85d1-3802ea5135b8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"28eca14cad8026ded241237500b7e886c19befd7e68a0e350c2b7677ab39efc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"2df5998d1c697533dcce76f5ca1a8dd2f315ee40cc155f9c15822e29d746be70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"b83badcc40496e121c06a8423ede3d7aeada8dd0924a4287300c02e0fa12d3e6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x4c656e042c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kbvm65vdfvv2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\",\n \"name\": \"f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:22.138065368Z\",\n \"endTime\": \"2026-10-12T16:34:29.402080119Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\n correlation ID: 96a806b3-e759-4bed-b150-87a5032cc614\n client request ID: e02662bb-1e5e-49c5-a613-eae526fdf63d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x4c656e042c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kbvm65vdfvv2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\",\n \"name\": \"f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:22.138065368Z\",\n \"endTime\": \"2026-10-12T16:34:29.402080119Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f732fbd5-1dc4-4651-a8ce-9bcd3363a01f\n correlation ID: 96a806b3-e759-4bed-b150-87a5032cc614\n client request ID: e02662bb-1e5e-49c5-a613-eae526fdf63d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2ea3b53369977a50f15744e668bd812581c016958002e7163d9ec0b7c3c1e7f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T16:22:14Z\n Ended: 2026-10-12T16:44:17Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9626368-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"3582b02e83aafb81bf733412b79a27fe625e754cddbe5f5185dfa95a89695507","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"47dea9fa6d473ca88a0d6daa3a8a61ce6d04ebe38764f2f66d84f10a437111c6","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3319f6bce300\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bqhxbgbszsk7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\",\n \"name\": \"9411d8dc-7343-4af4-8a8c-e09d46093966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.24212214Z\",\n \"endTime\": \"2026-10-12T16:34:19.71675375Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\n correlation ID: 83c55d94-e4f5-48dc-ab1d-2435b0924cd2\n client request ID: 5a6c1344-e9f4-4002-b03a-1f3a2c281b13\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3319f6bce300\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bqhxbgbszsk7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\",\n \"name\": \"9411d8dc-7343-4af4-8a8c-e09d46093966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.24212214Z\",\n \"endTime\": \"2026-10-12T16:34:19.71675375Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9411d8dc-7343-4af4-8a8c-e09d46093966\n correlation ID: 83c55d94-e4f5-48dc-ab1d-2435b0924cd2\n client request ID: 5a6c1344-e9f4-4002-b03a-1f3a2c281b13\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3a7b30534b9365b4cddb9d9b1732fc6fb74d3b9ec205c1369434404ac06c7329","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"844afb351725b6475ddd327ef78fbf5da706650b28665d4a6e378e60144b8e25","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x30d9dda2acc0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-24bgqfrw95w6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\",\n \"name\": \"de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:32.35534835Z\",\n \"endTime\": \"2026-10-12T16:33:39.538788418Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\n correlation ID: 84c7ec2b-83b1-4fd6-bc72-b09d6c16405b\n client request ID: 260e0b34-2d09-4902-9e8e-260359ca039a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x30d9dda2acc0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-24bgqfrw95w6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\",\n \"name\": \"de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:32.35534835Z\",\n \"endTime\": \"2026-10-12T16:33:39.538788418Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2f28c6-4bd8-443b-bc1a-dfe1125fba3d\n correlation ID: 84c7ec2b-83b1-4fd6-bc72-b09d6c16405b\n client request ID: 260e0b34-2d09-4902-9e8e-260359ca039a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"44c07cc23567fcb72c359d23e92fcece0bdb7579beed574173b1fb16f2b540c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"5c7543311511f19fcd39fd25b976d88fc18cf2a06f89b8cd4da4fdcb65f57861","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x26220088f600\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m6g5ktlgc8z7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\",\n \"name\": \"cc992b41-c033-41c6-80f2-695b7ffa7792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:06:22.806272123Z\",\n \"endTime\": \"2026-10-12T16:35:29.429759823Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\n correlation ID: 2df4db99-b25a-49b2-8229-5aef1853845b\n client request ID: 7993ac50-f843-4bcf-a848-7da89c1dfb9b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x26220088f600\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m6g5ktlgc8z7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\",\n \"name\": \"cc992b41-c033-41c6-80f2-695b7ffa7792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:06:22.806272123Z\",\n \"endTime\": \"2026-10-12T16:35:29.429759823Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc992b41-c033-41c6-80f2-695b7ffa7792\n correlation ID: 2df4db99-b25a-49b2-8229-5aef1853845b\n client request ID: 7993ac50-f843-4bcf-a848-7da89c1dfb9b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4b5887b3eb2b33b0d83b45fd73d0614d73d083e4de3e99cafac4b02a9c4e0904","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"5f344ec756d50afef29819a6882bec64ddbd8b7b94b117d33badf95b4228e5c9","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x313891a9c3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rc7jts7c5b7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\",\n \"name\": \"86abf47d-06c9-4a9f-8086-17c447bba1ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.029936909Z\",\n \"endTime\": \"2026-10-12T16:34:19.353815138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\n correlation ID: 2dc76f4c-c923-43f3-80f7-1614d5e972ef\n client request ID: 88764da1-36ee-45ce-bf37-b0a4af6df387\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x313891a9c3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rc7jts7c5b7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\",\n \"name\": \"86abf47d-06c9-4a9f-8086-17c447bba1ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.029936909Z\",\n \"endTime\": \"2026-10-12T16:34:19.353815138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86abf47d-06c9-4a9f-8086-17c447bba1ce\n correlation ID: 2dc76f4c-c923-43f3-80f7-1614d5e972ef\n client request ID: 88764da1-36ee-45ce-bf37-b0a4af6df387\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e5f2646eb0e8fce2cfeb384826c98db18e2a75063e65e46269d8e61d3204a29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"160071fa1d66581da10fab1353ab09d80ff5734975070cab9434ca0e1322ff49","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ff7db0580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tpdz2pzq6nx2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\",\n \"name\": \"899050a2-6fb7-479c-83ea-7740a027ce95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:06:23.186199816Z\",\n \"endTime\": \"2026-10-12T16:35:29.665151108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\n correlation ID: 12882895-7a3c-4953-aa64-66a3a89fcf92\n client request ID: ff7030da-8a27-4cf5-a404-898aa2f8f0b7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ff7db0580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tpdz2pzq6nx2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\",\n \"name\": \"899050a2-6fb7-479c-83ea-7740a027ce95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:06:23.186199816Z\",\n \"endTime\": \"2026-10-12T16:35:29.665151108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899050a2-6fb7-479c-83ea-7740a027ce95\n correlation ID: 12882895-7a3c-4953-aa64-66a3a89fcf92\n client request ID: ff7030da-8a27-4cf5-a404-898aa2f8f0b7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e90b453b72fc2d3cb5f0cd0d8d82f52b2722c41471c0f43b3cc16d29b19f880","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"51071ac504e6050a1fd884cd82764171a1d38ab7411e9ae858f536e87567f055","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-rc5bhp\" with version 4.20.41 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a5c448360\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rc5bhp: failed waiting for cluster=\"cluster-zstream-4-20-rc5bhp\" in resourcegroup=\"rg-zstream-upgrade-4-20-nfbcv4zwq6cm\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\",\n \"name\": \"590c7e12-8d58-422f-a728-bd866abfa3a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:04.337525362Z\",\n \"endTime\": \"2026-10-12T16:34:09.111845276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\n correlation ID: a248977c-0db4-411e-b79f-a5f86cd545ea\n client request ID: 92f67dce-a9c7-42a0-ad8e-4345b06da787\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-rc5bhp\" with version 4.20.41 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a5c448360\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rc5bhp: failed waiting for cluster=\"cluster-zstream-4-20-rc5bhp\" in resourcegroup=\"rg-zstream-upgrade-4-20-nfbcv4zwq6cm\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\",\n \"name\": \"590c7e12-8d58-422f-a728-bd866abfa3a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:04.337525362Z\",\n \"endTime\": \"2026-10-12T16:34:09.111845276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/590c7e12-8d58-422f-a728-bd866abfa3a0\n correlation ID: a248977c-0db4-411e-b79f-a5f86cd545ea\n client request ID: 92f67dce-a9c7-42a0-ad8e-4345b06da787\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4f6b106047494f6a5e368897c1d2d6374aaa31f1607927048b1d6452a1fed186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"44ded8bb1a8d8a2ac19d80e715a411a4b8efc7b7cc9f887538780c1e34b93ab9","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b8d1b766900\u003e: \n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\",\n \"name\": \"f36d9cc8-15b8-4ef7-9552-b1692d74dae3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:57.943824646Z\",\n \"endTime\": \"2026-10-12T16:34:59.426358813Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\n correlation ID: 0b896154-4688-484a-b4e5-ac1c715565ea\n client request ID: 6c21476c-0d7a-4112-bab1-e73b18ede350\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b8d1b766900\u003e: \n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\",\n \"name\": \"f36d9cc8-15b8-4ef7-9552-b1692d74dae3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:57.943824646Z\",\n \"endTime\": \"2026-10-12T16:34:59.426358813Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f36d9cc8-15b8-4ef7-9552-b1692d74dae3\n correlation ID: 0b896154-4688-484a-b4e5-ac1c715565ea\n client request ID: 6c21476c-0d7a-4112-bab1-e73b18ede350\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"52c9efa87ed63557eb7ea079e88fc304f1d0d69203a928c354e03b9fe9d571e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"a312b9a0ce9a609eef608f7ee363a1a717dc275fe1d0ca99a6b39591f5f7c10b","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qrw9cj with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x31c7526a720\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qrw9cj: failed waiting for cluster=\"np-version-upgrade-cluster-qrw9cj\" in resourcegroup=\"rg-np-version-upgrade-qrw9cj-lf88x46jszn6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\",\n \"name\": \"dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:32:34.07090627Z\",\n \"endTime\": \"2026-10-12T17:01:39.251657145Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\n correlation ID: bfd957bf-719e-49b3-bdac-7fd0797f19e3\n client request ID: 08772a89-5b8f-477e-91ad-296893cc14ba\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qrw9cj with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x31c7526a720\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qrw9cj: failed waiting for cluster=\"np-version-upgrade-cluster-qrw9cj\" in resourcegroup=\"rg-np-version-upgrade-qrw9cj-lf88x46jszn6\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\",\n \"name\": \"dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:32:34.07090627Z\",\n \"endTime\": \"2026-10-12T17:01:39.251657145Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc431aa9-1ae1-412b-b014-b6aeaafaa4d9\n correlation ID: bfd957bf-719e-49b3-bdac-7fd0797f19e3\n client request ID: 08772a89-5b8f-477e-91ad-296893cc14ba\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5af420b68c73ca7dd611dc9837f9574662aa864e25f4cef6974eb5a0ec7bcabc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"97fe8f81d50ab0c04cc31936087a1cb6ccf4abd2ff0897e188700ec58b85890f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8drrxw with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e21426be280\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8drrxw: failed waiting for cluster=\"np-version-upgrade-cluster-8drrxw\" in resourcegroup=\"rg-np-version-upgrade-8drrxw-2d9pwlfx6z2c\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\",\n \"name\": \"8202470b-a319-4568-8066-47a743d0153b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:32:31.241515725Z\",\n \"endTime\": \"2026-10-12T17:01:39.223309739Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\n correlation ID: 929ee1ce-8b15-4004-9f50-2422b86c1eea\n client request ID: 92329117-f931-4d63-9f18-2bd6e17d6193\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8drrxw with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e21426be280\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8drrxw: failed waiting for cluster=\"np-version-upgrade-cluster-8drrxw\" in resourcegroup=\"rg-np-version-upgrade-8drrxw-2d9pwlfx6z2c\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\",\n \"name\": \"8202470b-a319-4568-8066-47a743d0153b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:32:31.241515725Z\",\n \"endTime\": \"2026-10-12T17:01:39.223309739Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8202470b-a319-4568-8066-47a743d0153b\n correlation ID: 929ee1ce-8b15-4004-9f50-2422b86c1eea\n client request ID: 92329117-f931-4d63-9f18-2bd6e17d6193\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5bf662706c102afcdff94504dc0e6448bc92a2c998d4e5b00a8433c9ef77f368","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"6ab7f7a7d7318820073a94e892ad474901c481d146387431db8c950325a1baa6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-xqmd9dg8sc2s/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-xqmd9dg8sc2s/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"60231bca4a42ffc246538500e920fa883b2987d0a95fa02a7f20673f92056d2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"06d984b5bd61e16ead3a82aa893ca35dc9577f417c04093909481769e3f573b8","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"610b545e447a3773e8e78c5edd37793a133b58bc269b365dc5e36f46d365fb51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"c7ae04b38766210a8108f94bb04d957dfc5bca36ff0aa8754d485ec2530ed6a3","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cf76962bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f25877fzvj4h\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\",\n \"name\": \"65643201-e5f5-469d-8fa3-31adca10b028\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:29:46.48300317Z\",\n \"endTime\": \"2026-10-12T16:58:49.225784039Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\n correlation ID: 2d5e9971-8511-412a-9936-51793e57b7f8\n client request ID: d197ee43-d74a-43bb-ad30-5edf6b3f8b75\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cf76962bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f25877fzvj4h\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\",\n \"name\": \"65643201-e5f5-469d-8fa3-31adca10b028\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:29:46.48300317Z\",\n \"endTime\": \"2026-10-12T16:58:49.225784039Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65643201-e5f5-469d-8fa3-31adca10b028\n correlation ID: 2d5e9971-8511-412a-9936-51793e57b7f8\n client request ID: d197ee43-d74a-43bb-ad30-5edf6b3f8b75\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"68f32efe22309e5df318532a9062f5b57b838f43e99e5dbb30564abcf60303f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"4d85fc8c6214af114002c843ebdc86a94cf3126e2e1355356d3ebd73640aa90f","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x36802a1a8de0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mffj5zzpg42s\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\",\n \"name\": \"30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:03.29403013Z\",\n \"endTime\": \"2026-10-12T16:34:09.534630695Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\n correlation ID: b0972b4f-2e18-4424-bcbb-fe9ff318739b\n client request ID: ba2567e6-2c53-40b7-88ae-12adb3090f69\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x36802a1a8de0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mffj5zzpg42s\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\",\n \"name\": \"30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:03.29403013Z\",\n \"endTime\": \"2026-10-12T16:34:09.534630695Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b562f5-1652-49eb-acd3-5e2ba2ab5b2c\n correlation ID: b0972b4f-2e18-4424-bcbb-fe9ff318739b\n client request ID: ba2567e6-2c53-40b7-88ae-12adb3090f69\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"69b5906222e25a7932641db55eeba9f33f1376ebcc1958d0d5bc7a1d26e09d3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"dbf86b79040a8aaf8dd94a9133ea7c86d12cf15986aa135f1c6a41485545be9c","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x279567c40340\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-6lf4bkr5xqfk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\",\n \"name\": \"2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:30:06.89817477Z\",\n \"endTime\": \"2026-10-12T16:59:09.20400123Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\n correlation ID: 516be8ef-3151-4f6b-9ad6-def825112a39\n client request ID: 70377283-1a83-4ef0-bf32-cc22fa7b80a0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x279567c40340\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-6lf4bkr5xqfk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\",\n \"name\": \"2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:30:06.89817477Z\",\n \"endTime\": \"2026-10-12T16:59:09.20400123Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d49ef0a-2dbf-4175-b3ca-68a7b28d1c85\n correlation ID: 516be8ef-3151-4f6b-9ad6-def825112a39\n client request ID: 70377283-1a83-4ef0-bf32-cc22fa7b80a0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ac91ef2e7ff43b5b621ae65fba2cc51ba6febc0ccbeb55214eeb2a5dca0f897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"alert","signature_id":"f0a5ca335c5452170de0c7d7ae55e4a8594a4284ebc53b77ca227c096a57881b","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T17:06:14Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"822572330e763e1f481608af117dcac15b38f729bba42ea76bb1b52aa501d54e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"b2b126547a0bcacd7f8d6fadea91d4a58f4981406c524e6014bd0b7da6c92d4d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-4ztzcq7pqv5d/cluster-candidate-4-21-gjxhwz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x20aa34e90be0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gjxhwz: failed waiting for cluster=\"cluster-candidate-4-21-gjxhwz\" in resourcegroup=\"rg-candidate-4-21-4ztzcq7pqv5d\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\",\n \"name\": \"c3692a11-8775-42df-b3ca-63fd2bc4eff6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.496754125Z\",\n \"endTime\": \"2026-10-12T16:34:19.708354245Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\n correlation ID: ac77532e-4fc3-4e55-b685-8e5b46e539cb\n client request ID: ad2be3ef-6fea-4be1-89e4-a5c81c2beb73\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-4ztzcq7pqv5d/cluster-candidate-4-21-gjxhwz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x20aa34e90be0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gjxhwz: failed waiting for cluster=\"cluster-candidate-4-21-gjxhwz\" in resourcegroup=\"rg-candidate-4-21-4ztzcq7pqv5d\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\",\n \"name\": \"c3692a11-8775-42df-b3ca-63fd2bc4eff6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.496754125Z\",\n \"endTime\": \"2026-10-12T16:34:19.708354245Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3692a11-8775-42df-b3ca-63fd2bc4eff6\n correlation ID: ac77532e-4fc3-4e55-b685-8e5b46e539cb\n client request ID: ad2be3ef-6fea-4be1-89e4-a5c81c2beb73\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8522d0bb4e59d980572693747a0b1648f4d9eb65d1f8a327349284f81fbe440f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"6e1b7db0d8aeab353439c24ad4514730ee14c5e2353bca21858fe1f9675242cc","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xf88c4c20800\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-n8plk7fwndhf\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\",\n \"name\": \"a1fb4d6d-b979-4cd4-8635-598556293c82\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:28.784606371Z\",\n \"endTime\": \"2026-10-12T16:33:29.468218368Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\n correlation ID: c21803de-cafe-498c-9ca8-537b168f0464\n client request ID: e0f64d36-cf5a-405b-8e33-7500ab894b6a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xf88c4c20800\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-n8plk7fwndhf\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\",\n \"name\": \"a1fb4d6d-b979-4cd4-8635-598556293c82\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:28.784606371Z\",\n \"endTime\": \"2026-10-12T16:33:29.468218368Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1fb4d6d-b979-4cd4-8635-598556293c82\n correlation ID: c21803de-cafe-498c-9ca8-537b168f0464\n client request ID: e0f64d36-cf5a-405b-8e33-7500ab894b6a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"854eb1a5c98f35a4d2ef9c0cb3cefb53cfc3e78170e96917a35617a9eb4e0ab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"ed62ccd125102b89fe1e6cd6a42a65c596b410194665770b4c1ecb63e1ef3a7a","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x18e96d48740\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nmxmwv72jvsp\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\",\n \"name\": \"823a8af1-af0e-44ed-9ac4-a4eded671444\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.423039355Z\",\n \"endTime\": \"2026-10-12T16:34:19.36354313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\n correlation ID: 8547f930-92f7-43c6-a766-b1a655689fe6\n client request ID: 4816b204-7271-40c1-94ba-c3e04d5cd401\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x18e96d48740\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nmxmwv72jvsp\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\",\n \"name\": \"823a8af1-af0e-44ed-9ac4-a4eded671444\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.423039355Z\",\n \"endTime\": \"2026-10-12T16:34:19.36354313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823a8af1-af0e-44ed-9ac4-a4eded671444\n correlation ID: 8547f930-92f7-43c6-a766-b1a655689fe6\n client request ID: 4816b204-7271-40c1-94ba-c3e04d5cd401\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8a404bcaa8d0f866b86f2d038a72206a6597fef07e52095d1f0775e6c9bcb443","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9626368 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T16:24:31Z\n Ended: 2026-10-12T16:44:33Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}},{"row_id":"8ca364428a9afd438608ad47fa3111071562464e2366b1f69606dcb5c52a6730","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"7c6abafe393255f1739fdb387bd94e187d718daaaf8baf7ec69854b4ddf141b2","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xe49980b780\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bqcfl4xmn8nv\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\",\n \"name\": \"ca46772d-874d-464a-b1f4-1a692654e2d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:33.378694604Z\",\n \"endTime\": \"2026-10-12T16:33:39.38085323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\n correlation ID: 627a4075-ce28-4086-907b-f8bcd1c50f04\n client request ID: aca0242a-0a5c-452d-9295-e9fd4a074d56\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xe49980b780\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bqcfl4xmn8nv\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\",\n \"name\": \"ca46772d-874d-464a-b1f4-1a692654e2d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:33.378694604Z\",\n \"endTime\": \"2026-10-12T16:33:39.38085323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca46772d-874d-464a-b1f4-1a692654e2d4\n correlation ID: 627a4075-ce28-4086-907b-f8bcd1c50f04\n client request ID: aca0242a-0a5c-452d-9295-e9fd4a074d56\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"97b2984b0d8c879b319478d54e59061634efd6c230892338f56850645c608760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"bec2ded38a9a866918a5ba9ad1e9a064412687841aabfca8c59e91dc8567ad9c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x314814c9ae20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x314814c9ae20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded"}},{"row_id":"9ab5483da446107f52b22c7c0f5acd4bb95186258002ae9d3d9e44ee4ccf58eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"a0f1bf26985a99e6539af709b7c6cfaeed3b89c57f24089e9a344841e1966c77","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x12e83a5986a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jqt7djmh86jz\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\",\n \"name\": \"49dd0374-7176-4fef-8468-54280e5fe2f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.555692748Z\",\n \"endTime\": \"2026-10-12T16:34:19.574131662Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\n correlation ID: ea00b3c3-aa94-4a8e-8560-6b23011adbb3\n client request ID: 1b81d0e4-7d6e-4789-ae88-b45f2e9a808d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x12e83a5986a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jqt7djmh86jz\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\",\n \"name\": \"49dd0374-7176-4fef-8468-54280e5fe2f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:13.555692748Z\",\n \"endTime\": \"2026-10-12T16:34:19.574131662Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49dd0374-7176-4fef-8468-54280e5fe2f0\n correlation ID: ea00b3c3-aa94-4a8e-8560-6b23011adbb3\n client request ID: 1b81d0e4-7d6e-4789-ae88-b45f2e9a808d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ae20f28ccbe6a45b0a8e7267bdce8a8d85901526413c9427ba53bcede8bf7e58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"e642f26ea0a98605c1da7d415345036ed06aed0f4668bc7504474fbfa4f815a5","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:144]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x36a986f04ae0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ngrj2qbq5mrc\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\",\n \"name\": \"cbadb236-9e49-4d36-8684-284ddba4ed57\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:32.389348176Z\",\n \"endTime\": \"2026-10-12T16:33:39.478546952Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\n correlation ID: cac45e57-6d25-44b2-80de-c57fabdd7948\n client request ID: 00b03fd4-4840-40a0-934f-aa0300d6a4be\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:144]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x36a986f04ae0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ngrj2qbq5mrc\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\",\n \"name\": \"cbadb236-9e49-4d36-8684-284ddba4ed57\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:32.389348176Z\",\n \"endTime\": \"2026-10-12T16:33:39.478546952Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbadb236-9e49-4d36-8684-284ddba4ed57\n correlation ID: cac45e57-6d25-44b2-80de-c57fabdd7948\n client request ID: 00b03fd4-4840-40a0-934f-aa0300d6a4be\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b6093bd1d9f30473e59d1f51c7060fcb50bb6452f5b32c0556fbbe33827c8ad7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"1ec3f7dc73b39cc393755b265af0c51797463b27d90c21e0a2834ab16544dbf0","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2bb24c054ae0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-n65nc7rq5rdg\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\",\n \"name\": \"e42e9718-8312-4170-a28d-c30c09346551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:02.273005181Z\",\n \"endTime\": \"2026-10-12T16:34:09.206366816Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\n correlation ID: c61ae62a-a8ec-4dc7-bbc4-d1af2cd5934f\n client request ID: b3334a96-1e13-4945-873b-ebefe1b7128f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2bb24c054ae0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-n65nc7rq5rdg\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\",\n \"name\": \"e42e9718-8312-4170-a28d-c30c09346551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:02.273005181Z\",\n \"endTime\": \"2026-10-12T16:34:09.206366816Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e42e9718-8312-4170-a28d-c30c09346551\n correlation ID: c61ae62a-a8ec-4dc7-bbc4-d1af2cd5934f\n client request ID: b3334a96-1e13-4945-873b-ebefe1b7128f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b647c7e0631f98e1eae62630d85fcbc96f7fa4f5d1bb724907eb1e2fe7855194","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"61789678564be0aad54c9098dc019c63c0a07abf6db9351123669661958b90b1","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x18153c31e440\u003e: \n failed to create HCP cluster np-downgrade-mptb8x: failed waiting for cluster=\"np-downgrade-mptb8x\" in resourcegroup=\"rg-np-downgrade-mptb8x-2fb42qhh2sh4\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\",\n \"name\": \"5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:34:25.928266818Z\",\n \"endTime\": \"2026-10-12T17:03:29.209850456Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\n correlation ID: 6cef0923-9240-4826-b9dd-aed77d039af5\n client request ID: 7bed6fac-3b1e-4c42-843d-6588be72b449\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x18153c31e440\u003e: \n failed to create HCP cluster np-downgrade-mptb8x: failed waiting for cluster=\"np-downgrade-mptb8x\" in resourcegroup=\"rg-np-downgrade-mptb8x-2fb42qhh2sh4\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\",\n \"name\": \"5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:34:25.928266818Z\",\n \"endTime\": \"2026-10-12T17:03:29.209850456Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a4ba465-c77e-48a8-b1bf-fd5ce064f81a\n correlation ID: 6cef0923-9240-4826-b9dd-aed77d039af5\n client request ID: 7bed6fac-3b1e-4c42-843d-6588be72b449\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bc899a1a1de22e3e12902e2b9e35cd2855245e095b882c4fd8b2135df63c1f47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"586593e2fff2e69d645fc42dff24f41310743ceba1deccc3b657c96749c0928b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x14c939b3c680\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c7rpzcklv7b5\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\",\n \"name\": \"c785e249-3c26-40d1-a8a5-fad7e1a9cb69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:52.118812309Z\",\n \"endTime\": \"2026-10-12T16:34:00.247635722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\n correlation ID: 664c91b2-48cb-49a9-9227-c75515a8f336\n client request ID: 67a94d14-b66a-48b0-a648-850ba5342d39\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x14c939b3c680\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c7rpzcklv7b5\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\",\n \"name\": \"c785e249-3c26-40d1-a8a5-fad7e1a9cb69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:52.118812309Z\",\n \"endTime\": \"2026-10-12T16:34:00.247635722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c785e249-3c26-40d1-a8a5-fad7e1a9cb69\n correlation ID: 664c91b2-48cb-49a9-9227-c75515a8f336\n client request ID: 67a94d14-b66a-48b0-a648-850ba5342d39\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c2748b12c784a06657fb7d1c7cbf5c958fdf0b89a5a178d5e551e359b5f2566d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"7cb425f2d7951985395b91345c374be4c0dca4d08831fcfb07f410b4baec6807","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x31377de45e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sslgcqs6f9c9\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\",\n \"name\": \"417c11d2-2f9b-476e-aa7b-d131547953b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:07:17.197387737Z\",\n \"endTime\": \"2026-10-12T16:36:19.074391072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\n correlation ID: aa74bff6-62d4-412b-8a0b-e853b2cfa1b8\n client request ID: fed0be28-4bc7-4d65-87e8-2d4e1db112b1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x31377de45e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sslgcqs6f9c9\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\",\n \"name\": \"417c11d2-2f9b-476e-aa7b-d131547953b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:07:17.197387737Z\",\n \"endTime\": \"2026-10-12T16:36:19.074391072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/417c11d2-2f9b-476e-aa7b-d131547953b6\n correlation ID: aa74bff6-62d4-412b-8a0b-e853b2cfa1b8\n client request ID: fed0be28-4bc7-4d65-87e8-2d4e1db112b1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c4a9ebe361829b16b1ce8a17611c6959eda825424ee70be552bfa0745ca49824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"9f58aa2dc15f7e7336a495843cc0f699c3e5b143f75bf07f3842358d658bcdca","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--k9r29d9g92h9/cluster-hs-pre-4lqcqr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x29b422d704c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4lqcqr: failed waiting for cluster=\"cluster-hs-pre-4lqcqr\" in resourcegroup=\"rg-hs-pre--k9r29d9g92h9\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\",\n \"name\": \"ab948028-30ab-4aa6-8829-3a06cc7c1dde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.199986974Z\",\n \"endTime\": \"2026-10-12T16:34:19.441219246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\n correlation ID: e66c132f-da52-4c57-936b-c0afdc1a831a\n client request ID: 0d1874a1-8e88-473d-8ec4-fe6ab6c34f8f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--k9r29d9g92h9/cluster-hs-pre-4lqcqr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x29b422d704c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4lqcqr: failed waiting for cluster=\"cluster-hs-pre-4lqcqr\" in resourcegroup=\"rg-hs-pre--k9r29d9g92h9\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\",\n \"name\": \"ab948028-30ab-4aa6-8829-3a06cc7c1dde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.199986974Z\",\n \"endTime\": \"2026-10-12T16:34:19.441219246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab948028-30ab-4aa6-8829-3a06cc7c1dde\n correlation ID: e66c132f-da52-4c57-936b-c0afdc1a831a\n client request ID: 0d1874a1-8e88-473d-8ec4-fe6ab6c34f8f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9068eee23685f231c9bc48867c968591cc849b102bc3f138f9def0bea8ff668","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"59da2ca63ee261235a31615de260be397d214b0f4f694b160b2ac972a17ed31e","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27aef5ac99f567baefaad85029e0d3da9b4dbe40c6bbe0be985920a793730d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ce7cfa0122fdabb043a75ae8fb2c618d9e6454b1b88371987692035043997777","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"434f1f50bc3fec5a453eb5fc6ce070a5ad6cf40f73119f0ef622c86873aa56ea","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xa246b118700\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lv4p56sl2gn5\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\",\n \"name\": \"a11d3580-395c-4b4e-aba8-1f155f029317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:38:07.657971903Z\",\n \"endTime\": \"2026-10-12T17:07:09.209513313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\n correlation ID: 3a5bde5c-ec4f-4189-9bf7-ce8f83338ce6\n client request ID: 628bdfcd-f4ac-4fd8-bde6-db5ebf05dcd7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xa246b118700\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lv4p56sl2gn5\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\",\n \"name\": \"a11d3580-395c-4b4e-aba8-1f155f029317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:38:07.657971903Z\",\n \"endTime\": \"2026-10-12T17:07:09.209513313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a11d3580-395c-4b4e-aba8-1f155f029317\n correlation ID: 3a5bde5c-ec4f-4189-9bf7-ce8f83338ce6\n client request ID: 628bdfcd-f4ac-4fd8-bde6-db5ebf05dcd7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf8af67b242f332d6c3481c5390716c3cd5106f7b80d82c4733ce8e20a9076ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"40e270b1250c572d95ad297676751fd03a52912e32aa17a878d7239f4fb1933c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x50023914b40\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-t9tsfck7gs88\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\",\n \"name\": \"1269e331-6a56-4e39-8d97-876562b4239c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:30:45.872172093Z\",\n \"endTime\": \"2026-10-12T16:59:49.217488716Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\n correlation ID: b81a6b2a-780b-4e63-aa63-4a20441c7bb1\n client request ID: bb9cfec1-df9f-4c19-af52-b64f941d0cea\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x50023914b40\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-t9tsfck7gs88\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\",\n \"name\": \"1269e331-6a56-4e39-8d97-876562b4239c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:30:45.872172093Z\",\n \"endTime\": \"2026-10-12T16:59:49.217488716Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1269e331-6a56-4e39-8d97-876562b4239c\n correlation ID: b81a6b2a-780b-4e63-aa63-4a20441c7bb1\n client request ID: bb9cfec1-df9f-4c19-af52-b64f941d0cea\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d39adf2f4d16ca97bd6bd27d8dc3c75d58b44cc9b8e1e482006b9a1dff8e6db4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"f29a7407248c09d9f2d63fc02a535acb501328e35c4bfacc81f04e0e98c0e854","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"d4a6d957c65b9499be99348d7259d168136cb3e7aef54d527904bd40d57d9dfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"0a7b025a664cd937a27c1586ab0853b5c1ef3166363dadc1edebbd26f45f0f59","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d5643cd9ccc65ef1c736b21f476da417b59681ee92ca6f7be2f9c2df58749fba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"e72a8b419adb6b53b3d96517db9d967450746716b4acc806c6d35e614fc4cc81","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n {\n errs: [\n \u003c*fmt.wrapError | 0x11d596138740\u003e{\n msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n },\n },\n \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n {\n errs: [\n \u003c*fmt.wrapError | 0x11d596138740\u003e{\n msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n },\n },\n \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"d6be27d50464c3c12fb597b52803596bd581e4593a32fe8dfd6424f7861f8f78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"ed6cd23384ef866a46342ced82dc3a93106a5447a643b0459d4e1595765f54a8","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b5eb3004540\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-x4ktbm22q9kc\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\",\n \"name\": \"de0f0e5b-f214-4efd-afd5-7985bdb11b91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:31:56.54495042Z\",\n \"endTime\": \"2026-10-12T17:00:59.212681854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\n correlation ID: 3f6e83f7-ab63-4f4a-8e9b-190aaafac047\n client request ID: fa2bb10b-d022-4a5a-a5a4-67ed02cc2bf5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b5eb3004540\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-x4ktbm22q9kc\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\",\n \"name\": \"de0f0e5b-f214-4efd-afd5-7985bdb11b91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:31:56.54495042Z\",\n \"endTime\": \"2026-10-12T17:00:59.212681854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de0f0e5b-f214-4efd-afd5-7985bdb11b91\n correlation ID: 3f6e83f7-ab63-4f4a-8e9b-190aaafac047\n client request ID: fa2bb10b-d022-4a5a-a5a4-67ed02cc2bf5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d8cc69080efa959086fbaa7aeccde542268a5d3279fbcdf5c6ec8aa665181660","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"9f01d929e13b36781c18a8d7f12e2b1fcb9913e0a3fcb25d7d5ff885c820a357","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b93509c09c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n45dqcs4k8ll\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\",\n \"name\": \"f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:05.265108129Z\",\n \"endTime\": \"2026-10-12T16:34:09.122414349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\n correlation ID: 17c8647a-3930-47ce-877f-ab9387c5b822\n client request ID: 26bd2508-f678-4251-89af-cb812e799893\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b93509c09c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n45dqcs4k8ll\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\",\n \"name\": \"f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:05.265108129Z\",\n \"endTime\": \"2026-10-12T16:34:09.122414349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8ace9bf-48b3-4134-a3b6-ff7e0e2f991f\n correlation ID: 17c8647a-3930-47ce-877f-ab9387c5b822\n client request ID: 26bd2508-f678-4251-89af-cb812e799893\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df09fd44e63b01f39191f1937f86dda0c5fee8bd6a37456ffc3bb86b9686f572","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"alert","signature_id":"2ac18c0f28c3db988430e8cd508afbb6dc2ac259140c6a1cba13d912134a5113","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-56bb7dd747-h9s6h\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-56bb7dd747-h9s6h (namespace aro-hcp) on cluster ci01-j9626368-svc has averaged 134.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:54:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-c6ff9b6f4-b8fh5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-c6ff9b6f4-b8fh5 (namespace kube-applier) on cluster ci01-j9626368-mgmt-2 has averaged 150.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:56:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-65fd757c56-d7zhs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-65fd757c56-d7zhs (namespace kube-applier) on cluster ci01-j9626368-mgmt-1 has averaged 164.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"df42395d5ee72ff30f31a9dae9485cafb72b61ff6c70aaae5b1088ff0887c14b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"85acf0e937ffe0027b15bd44eca4c7c42194560ce1678ed387a13da6d7aadcb6","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c903d6aab20\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2cx7cfr7wjgf\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\",\n \"name\": \"24f3c84d-3b9d-478d-8bea-d6abeb4b5683\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:33.568522156Z\",\n \"endTime\": \"2026-10-12T16:33:39.594289919Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\n correlation ID: 29f8dc5f-71ff-4129-9695-5297503fc030\n client request ID: eb69571c-de2f-4fcf-9242-baf4d64dae94\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c903d6aab20\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2cx7cfr7wjgf\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\",\n \"name\": \"24f3c84d-3b9d-478d-8bea-d6abeb4b5683\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:33.568522156Z\",\n \"endTime\": \"2026-10-12T16:33:39.594289919Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f3c84d-3b9d-478d-8bea-d6abeb4b5683\n correlation ID: 29f8dc5f-71ff-4129-9695-5297503fc030\n client request ID: eb69571c-de2f-4fcf-9242-baf4d64dae94\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e027556f6ae5c7cb95b675a6e37b932bda28140c5ed8e882d13b6a3b63ec352f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"d1805d4cf77820ef4f29d07237f0247a441e0228a2174ebcae5ceacb2de61aa9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a02392b4920\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-cfbbzz7dz6nz\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\",\n \"name\": \"81f05566-7fcf-446c-a1f9-83457d6772d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:39:24.299369974Z\",\n \"endTime\": \"2026-10-12T17:08:29.200092329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\n correlation ID: f8f32452-a240-481d-801c-3f64f3fe12ff\n client request ID: 1ecb0ea9-2b8f-4d95-b795-94f372eb3eea\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a02392b4920\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-cfbbzz7dz6nz\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\",\n \"name\": \"81f05566-7fcf-446c-a1f9-83457d6772d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:39:24.299369974Z\",\n \"endTime\": \"2026-10-12T17:08:29.200092329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f05566-7fcf-446c-a1f9-83457d6772d4\n correlation ID: f8f32452-a240-481d-801c-3f64f3fe12ff\n client request ID: 1ecb0ea9-2b8f-4d95-b795-94f372eb3eea\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e40500e1abd95e3629b30d8a534b2deb2f1c3f6cbb0f256ad6e8b8f07d25a6c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"alert","signature_id":"d53a2236830f49d3e673f190f6889ac45bcba408a0f6c004202adfc7dcecbe92","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T17:20:12Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"e7774948fc34059cf764f41ee947a6e1d3551352f7dc3de89a127a61091fdc49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"055f9a9b6c244ec70223014863e02389b9a479621a6d56cd84fc1a590883325e","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ea65c9a21928e09fb9a9ef1e37b4e988885fe1770566e9c7a3cc75562a6e6f8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"8181f041b308c072f1833b4eb56517c581e5c8e4e9cd0a8ea0e6808fccc7a57f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x8d9ed1ecf40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7lbnvkdmwzbl\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\",\n \"name\": \"9e62beee-100e-4e5e-91cf-7bb6624ff2a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:56.267179906Z\",\n \"endTime\": \"2026-10-12T16:34:59.200522398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\n correlation ID: 2520b360-0d36-4ca0-bb12-eef341706d5b\n client request ID: 205b5e74-f26c-4c8a-80fe-6cd33ba159fc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x8d9ed1ecf40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7lbnvkdmwzbl\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\",\n \"name\": \"9e62beee-100e-4e5e-91cf-7bb6624ff2a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:56.267179906Z\",\n \"endTime\": \"2026-10-12T16:34:59.200522398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e62beee-100e-4e5e-91cf-7bb6624ff2a0\n correlation ID: 2520b360-0d36-4ca0-bb12-eef341706d5b\n client request ID: 205b5e74-f26c-4c8a-80fe-6cd33ba159fc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f0ff2fe6bc5c3121cd4ddd814eb17e121d8dd1765f085c702b7a8dfbf5326436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"b7d3cff9604baa7adb7447f8feddb9b4e1f30933ea4005d1bf550006ccbe8684","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f41f431c220\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-94qwhdgdlfld\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\",\n \"name\": \"bc606312-a6f8-4a33-bf01-c5d936147d70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:15.532663446Z\",\n \"endTime\": \"2026-10-12T16:34:19.695494446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\n correlation ID: 32984428-9d46-4a6b-8619-1d1460e2ac1b\n client request ID: b088058d-962b-4eb2-9131-24b6f9ceeb11\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f41f431c220\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-94qwhdgdlfld\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\",\n \"name\": \"bc606312-a6f8-4a33-bf01-c5d936147d70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:15.532663446Z\",\n \"endTime\": \"2026-10-12T16:34:19.695494446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc606312-a6f8-4a33-bf01-c5d936147d70\n correlation ID: 32984428-9d46-4a6b-8619-1d1460e2ac1b\n client request ID: b088058d-962b-4eb2-9131-24b6f9ceeb11\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f1062013333a68977812c6d11ae481afc3c6d952c73cb7ca94b84f9e0e01b722","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"e76fe38f94cc0c24f57a2b14af3523d40d7b5c15cd0e3fefac98dc8b85b99bed","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1720c4528160\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-248bgmphdnf7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\",\n \"name\": \"437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.4831266Z\",\n \"endTime\": \"2026-10-12T16:34:19.148385019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\n correlation ID: 776bda98-e7b3-4551-a040-3ba05f709a45\n client request ID: e18b8bd2-0ab9-4a4b-af54-2fdf8b5f8e56\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1720c4528160\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-248bgmphdnf7\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\",\n \"name\": \"437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.4831266Z\",\n \"endTime\": \"2026-10-12T16:34:19.148385019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/437b47ce-9c3f-4407-aeb1-fd10ad4d82e0\n correlation ID: 776bda98-e7b3-4551-a040-3ba05f709a45\n client request ID: e18b8bd2-0ab9-4a4b-af54-2fdf8b5f8e56\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f16a87b8cfa7fe7ce80cbd07ec97bfd4c858a8f2caeaaa3cd9084898428b0688","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"da295e3f891ebd53e53b834c83d5f2d2d8c7990e91838251fdcf5881a2a1f90a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-8h9snt\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x278858e785a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8h9snt: failed waiting for cluster=\"cluster-zstream-4-21-8h9snt\" in resourcegroup=\"rg-zstream-upgrade-4-21-k45g2vrqv5gs\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\",\n \"name\": \"a71975b4-9124-4e36-a3ce-5b86ec172675\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:38.921451129Z\",\n \"endTime\": \"2026-10-12T16:33:39.476865521Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\n correlation ID: f8e3e525-426e-44f2-88ff-9793dde001bf\n client request ID: 34463a57-8054-4c90-a50e-2c4ca0477092\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-8h9snt\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x278858e785a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8h9snt: failed waiting for cluster=\"cluster-zstream-4-21-8h9snt\" in resourcegroup=\"rg-zstream-upgrade-4-21-k45g2vrqv5gs\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\",\n \"name\": \"a71975b4-9124-4e36-a3ce-5b86ec172675\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:38.921451129Z\",\n \"endTime\": \"2026-10-12T16:33:39.476865521Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a71975b4-9124-4e36-a3ce-5b86ec172675\n correlation ID: f8e3e525-426e-44f2-88ff-9793dde001bf\n client request ID: 34463a57-8054-4c90-a50e-2c4ca0477092\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fd4935cccd696203e554cd8d667c4dad73677eb92ec298cd65d7ce472e39e28d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","failed_at":"e2e","signature_id":"945c8b8ba229efce78c82c55117d9808b09ecb3e3f5b4cd8cda0f81bc283543f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d4ffd7a600\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dvr2wr9tkktb\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\",\n \"name\": \"b0d18522-dc69-4e01-bb81-5507a89165e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:53.484267787Z\",\n \"endTime\": \"2026-10-12T16:34:59.431372216Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\n correlation ID: 632c6de6-daf9-402e-9574-3d088d679fc4\n client request ID: 1cc71eaf-6c2d-4d17-abdc-cf5c9a38f3ec\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d4ffd7a600\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dvr2wr9tkktb\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\",\n \"name\": \"b0d18522-dc69-4e01-bb81-5507a89165e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:53.484267787Z\",\n \"endTime\": \"2026-10-12T16:34:59.431372216Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d18522-dc69-4e01-bb81-5507a89165e0\n correlation ID: 632c6de6-daf9-402e-9574-3d088d679fc4\n client request ID: 1cc71eaf-6c2d-4d17-abdc-cf5c9a38f3ec\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":53,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","27aef5ac99f567baefaad85029e0d3da9b4dbe40c6bbe0be985920a793730d4f","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":53,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662371940143104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"open","pr_sha":"a6abbc2ac78ce68de467738122cc9b96a313a01f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T15:07:59Z","started_at":"2026-10-12T15:07:59Z","completed_at":"2026-10-12T15:15:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:16:28.107909424Z","timing_metadata_checked_at":"2026-10-12T17:16:28.107909424Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-12T17:16:28.111103215Z","region_metadata_checked_at":"2026-10-12T18:19:53.142970911Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3f3efb323da3b164c68e88716ed16a05e27196ef0b622e653168bbfbfaa42dde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662371940143104","occurred_at":"2026-10-12T15:07:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7383,"pr_state":"open","pr_sha":"f3c17b0fa5921de83f5edfb6810bffe5f0fd3b15","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T15:06:56Z","started_at":"2026-10-12T15:06:56Z","completed_at":"2026-10-12T17:41:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:48:19.1359772Z","timing_metadata_checked_at":"2026-10-12T17:48:19.1359772Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:48:19.138708991Z","region_metadata_checked_at":"2026-10-12T17:48:19.138708991Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5921d458df26e88569b783a63479c88e3ecc490f34d1b101b86522844e5fde8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","failed_at":"e2e","signature_id":"93e4587d665844128088f4ce3a0a7ac0670bc2eccb4898d241b67e1ee37e48d9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c6e11cbb74bffe980698f0ac18939b0cc44bb682a44aa8b5b4f43aee00cd17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"73d8a79251e50cc6b14902d650fd7ab6e0346824ff9c1d00abc54056550dfde0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","failed_at":"e2e","signature_id":"74adfdd35c7ce8eb88d441864d6b1e0ec991a2d255378a9ffce5b6cd5cf8549e","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.009s.\ncluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.009s.\ncluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["64c6e11cbb74bffe980698f0ac18939b0cc44bb682a44aa8b5b4f43aee00cd17","bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660658009116672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7467,"pr_state":"open","pr_sha":"395ae2305c3515cbbbe03da6d1c4c686fc44b301","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T15:00:29Z","started_at":"2026-10-12T15:00:29Z","completed_at":"2026-10-12T17:37:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:48:19.34809895Z","timing_metadata_checked_at":"2026-10-12T17:48:19.34809895Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:48:19.350362043Z","region_metadata_checked_at":"2026-10-12T17:48:19.350362043Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6bf95a8096665d8e3114f66f288391c0322cfbb78590407f13382498cb019a25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660658009116672","occurred_at":"2026-10-12T15:00:29Z","failed_at":"e2e","signature_id":"f0b4f980a6b881fe20a87e1354e5792bdbf23c290ec125adfa5d9d993ae815ea","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23913a80e730\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23913a80e730\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"open","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T14:59:35Z","started_at":"2026-10-12T14:59:35Z","completed_at":"2026-10-12T17:12:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:27:09.809340375Z","timing_metadata_checked_at":"2026-10-12T17:27:09.809340375Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:27:09.812341066Z","region_metadata_checked_at":"2026-10-12T17:27:09.812341066Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"79062af791593691ef5234fa9f2bed174b61a5dfc78078a22a74a35d7a51221c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","failed_at":"alert","signature_id":"36ccc2be84055641dbb6036217141bc4c104ca4b184f2aa2e64ec66e411c6dbe","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T16:16:01Z\n Ended: 2026-10-12T16:22:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0290944-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdbebgjal25ks2eu7kru50siu2d9fk7-d5m6i4f8v4g8r0f\", pod=\"kube-controller-manager-655fcf6674-rv7x7\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdbebgjal25ks2eu7kru50siu2d9fk7-d5m6i4f8v4g8r0f/kube-controller-manager-655fcf6674-rv7x7 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"df553447906b4086545f766dedf48e54a2d3a00927d79372f2b6e0a0159a5ce8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","failed_at":"e2e","signature_id":"222aacc360be62a53a3a25918371654d8976e4d02348bfbb0f85f02afd94062c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109659008674566144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"open","pr_sha":"04963132463344563688ac39d6d859cad4061591","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T14:54:37Z","started_at":"2026-10-12T14:54:37Z","completed_at":"2026-10-12T17:18:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:37:58.773359823Z","timing_metadata_checked_at":"2026-10-12T17:37:58.773359823Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:37:58.776775712Z","region_metadata_checked_at":"2026-10-12T17:37:58.776775712Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"946aa8408a001b2941d1a57cf414b15191d8ee78529fcfe68cecccd5f245d3f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109659008674566144","occurred_at":"2026-10-12T14:54:37Z","failed_at":"e2e","signature_id":"aa1b645215452d614b0ab2fb286d0f4c2a52df3c3bf4b0684396494f99dc7271","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x3aa92878a060\u003e: \n resource group \"hcp-metrics-w7n8h467qbnb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-w7n8h467qbnb\" to finish deleting: GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\",\n \"name\": \"eb261c81-ca83-45b2-bdfd-0de40227c103\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:58:20.608938335Z\",\n \"endTime\": \"2026-10-12T16:22:21.07258074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5jcbgo4cpr0su6el4fc6i0lag0dj still exists (deletion dispatched at 2026-10-12T15:58:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n correlation ID: 9da4d8e2-ebfd-4094-bf43-c7701d90768a\n client request ID: 61ad067a-2c1b-43a6-b798-69bd268d4471\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x3aa92878a060\u003e: \n resource group \"hcp-metrics-w7n8h467qbnb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-w7n8h467qbnb\" to finish deleting: GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\",\n \"name\": \"eb261c81-ca83-45b2-bdfd-0de40227c103\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:58:20.608938335Z\",\n \"endTime\": \"2026-10-12T16:22:21.07258074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5jcbgo4cpr0su6el4fc6i0lag0dj still exists (deletion dispatched at 2026-10-12T15:58:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n correlation ID: 9da4d8e2-ebfd-4094-bf43-c7701d90768a\n client request ID: 61ad067a-2c1b-43a6-b798-69bd268d4471\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T14:53:34Z","started_at":"2026-10-12T14:53:34Z","completed_at":"2026-10-12T17:17:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:37:58.673670141Z","timing_metadata_checked_at":"2026-10-12T17:37:58.673670141Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:37:58.677293429Z","region_metadata_checked_at":"2026-10-12T17:37:58.677293429Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"236571c4f42526317075935af38ebfe418751c8737c1268871bb83669666b75b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","failed_at":"e2e","signature_id":"9578272302c49c5a63e06e79a95c961c925c637a881235d46f047f8f61f08363","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4cmzlj277gjf/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4cmzlj277gjf/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"49eb9f1981d9c9898420fd20934ce46971a366dad66237d477dc3f97b0955e3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","failed_at":"e2e","signature_id":"b46b21369678c0c798aca134326c4166ed0c77d047a753de7c68744c3f55b704","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x151218b082b8\u003e: \n resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x151216d07800\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x151218b082b8\u003e: \n resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x151216d07800\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded"}},{"row_id":"602e1f22c1afeade4a459e3452712b662cdab06b53878d5266b07cb330b9f70c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","failed_at":"e2e","signature_id":"4ea81d2e607f663f33592f25d4328ddf786e7ed343f4f03175a81981d5aeb666","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x643438c6138\u003e: \n resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x643438c6138\u003e: \n resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"663335e2ff234db438f252722a5532157629f31c934cd990ce2182643911d3d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","failed_at":"e2e","signature_id":"c8fe87bb1fbea1d2b82daec5c2abcf5f379166a6c5ecbc60a09403cacb34657b","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x351b8e4ae138\u003e: \n resource group \"disabled-image-registry-pzs7vs68g9xt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-pzs7vs68g9xt\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\",\n \"name\": \"f1e860d3-9723-4468-b91f-a96474515d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:57:28.987630883Z\",\n \"endTime\": \"2026-10-12T16:21:31.7469599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5a4g859cjm0hvqmru36k5u916p18 still exists (deletion dispatched at 2026-10-12T15:57:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n correlation ID: eab286f4-49cc-48ea-8ec4-6c70b43ff1d0\n client request ID: 1a74a385-2e63-4deb-a488-b16ad26cc354\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x351b8e4ae138\u003e: \n resource group \"disabled-image-registry-pzs7vs68g9xt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-pzs7vs68g9xt\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\",\n \"name\": \"f1e860d3-9723-4468-b91f-a96474515d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:57:28.987630883Z\",\n \"endTime\": \"2026-10-12T16:21:31.7469599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5a4g859cjm0hvqmru36k5u916p18 still exists (deletion dispatched at 2026-10-12T15:57:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n correlation ID: eab286f4-49cc-48ea-8ec4-6c70b43ff1d0\n client request ID: 1a74a385-2e63-4deb-a488-b16ad26cc354\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"69575bdc0226ee7ad85fa5124d0e61f1fddf9ec29058d8e9e644c2b8f4a1709b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","failed_at":"e2e","signature_id":"9464cf8289c40c4ace5feeaa2af2e60f2ff16f8a6b1306bce054078519a3b607","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x1476512fa078\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n \"name\": \"d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:58:47.33491695Z\",\n \"endTime\": \"2026-10-12T16:22:51.509357112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb58jqqdu8adgelm46n5haqq88idqc still exists (deletion dispatched at 2026-10-12T15:58:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n correlation ID: f7d4d296-e895-4942-ace3-b59e68657889\n client request ID: 314b1f02-1204-4002-a941-604f43078239\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x1476512fa078\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n \"name\": \"d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:58:47.33491695Z\",\n \"endTime\": \"2026-10-12T16:22:51.509357112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb58jqqdu8adgelm46n5haqq88idqc still exists (deletion dispatched at 2026-10-12T15:58:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n correlation ID: f7d4d296-e895-4942-ace3-b59e68657889\n client request ID: 314b1f02-1204-4002-a941-604f43078239\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109657273612636160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7311,"pr_state":"open","pr_sha":"f51faacc521cc62f163b7fa835f6c2a1d64fe2e5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T14:47:36Z","started_at":"2026-10-12T14:47:36Z","completed_at":"2026-10-12T17:12:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:27:09.715636861Z","timing_metadata_checked_at":"2026-10-12T17:27:09.715636861Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:27:09.718946351Z","region_metadata_checked_at":"2026-10-12T17:27:09.718946351Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4f9b9579c79ab00798e867050260c9bd2cc09dedbea867fa8489d0232301e589","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109657273612636160","occurred_at":"2026-10-12T14:47:36Z","failed_at":"e2e","signature_id":"8a2cb86eeaaac2be5c2b729648eafebf2a78774b306792068dd64d9c83e1f663","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xaa70fc27a80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pcvlks6fdkqz\" to finish creating: GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n \"name\": \"26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:23.241617081Z\",\n \"endTime\": \"2026-10-12T16:21:32.102166958Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-q4ttl-4vcvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-pcvlks6fdkqz--managed/ingress-cert-le-np-1-q4ttl-4vcvt-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-q4ttl-4vcvt-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: RetryableError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"RetryableError\\\",\\n \\\"message\\\": \\\"A retryable error occurred.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n correlation ID: bfc832f2-b15d-4a34-8c01-edecd304fc1a\n client request ID: cd0488d8-da8e-467d-8248-7320c5456b91\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xaa70fc27a80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pcvlks6fdkqz\" to finish creating: GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n \"name\": \"26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:23.241617081Z\",\n \"endTime\": \"2026-10-12T16:21:32.102166958Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-q4ttl-4vcvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-pcvlks6fdkqz--managed/ingress-cert-le-np-1-q4ttl-4vcvt-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-q4ttl-4vcvt-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: RetryableError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"RetryableError\\\",\\n \\\"message\\\": \\\"A retryable error occurred.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n correlation ID: bfc832f2-b15d-4a34-8c01-edecd304fc1a\n client request ID: cd0488d8-da8e-467d-8248-7320c5456b91","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109655959428141056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"18c49d47df569da3e4d3a1f7894bd9628f0dae73","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T14:42:30Z","started_at":"2026-10-12T14:42:30Z","completed_at":"2026-10-12T16:58:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T17:06:17.160148926Z","timing_metadata_checked_at":"2026-10-12T17:06:17.160148926Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T17:06:17.166097408Z","region_metadata_checked_at":"2026-10-12T17:06:17.166097408Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0a6032ba834e154b978e1bda63bee1959df26a38add98bbcfde8a7d102160816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109655959428141056","occurred_at":"2026-10-12T14:42:30Z","failed_at":"e2e","signature_id":"0b00e0911be997c70989e7dd3678d4edc17e5aca2336e84d6fe6cf814fadeb28","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109643872534532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"open","pr_sha":"2dd7b8bb412fae02a46ebd9c6a31fa5276a5bcd4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T13:54:09Z","started_at":"2026-10-12T13:54:09Z","completed_at":"2026-10-12T16:06:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T16:14:03.492858556Z","timing_metadata_checked_at":"2026-10-12T16:14:03.492858556Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T16:14:03.495834046Z","region_metadata_checked_at":"2026-10-12T16:14:03.495834046Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7929845846f17fbaa10aa40bc5a20764b450ca4d5c8c5770ea3a61c9f377ec05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109643872534532096","occurred_at":"2026-10-12T13:54:09Z","failed_at":"e2e","signature_id":"1f4d7a78c6e6f2b64bfcac7b4fc7dadaf9ee3793c7757ef0699936b0dd1911ad","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x72d88398300\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x72d88398300\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109635102249062400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7376,"pr_state":"open","pr_sha":"521d3fc4b21cde036d1510f6fdc6897c0b831735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T13:19:38Z","started_at":"2026-10-12T13:19:38Z","completed_at":"2026-10-12T15:47:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:53:14.647568969Z","timing_metadata_checked_at":"2026-10-12T15:53:14.647568969Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:53:14.650505461Z","region_metadata_checked_at":"2026-10-12T15:53:14.650505461Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109631926875721728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7466,"pr_state":"closed","pr_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","final_merged_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T13:07:01Z","started_at":"2026-10-12T13:07:01Z","completed_at":"2026-10-12T15:14:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:21:22.271265159Z","timing_metadata_checked_at":"2026-10-12T15:21:22.271265159Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:21:22.273834352Z","region_metadata_checked_at":"2026-10-12T15:21:22.273834352Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"21c5097072b27775aeb5bfa1e803499de79c8013","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T12:58:13Z","started_at":"2026-10-12T12:58:13Z","completed_at":"2026-10-12T15:30:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:31:37.604024877Z","timing_metadata_checked_at":"2026-10-12T15:31:37.604024877Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:31:37.607176268Z","region_metadata_checked_at":"2026-10-12T15:31:37.607176268Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1e41a8de528e6077bfb1a45ce879224d53d1c349853b295901505751952e9920","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","failed_at":"alert","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T14:00:36Z\n Ended: 2026-10-12T14:17:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4606592-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T14:00:36Z\n Ended: 2026-10-12T14:17:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4606592-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"bae6591fa91425779719168f7b7eb2a16ce85a033bfb739f222104a0e3b64274","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","failed_at":"e2e","signature_id":"b12b641e87320622b3d715665e73509b0e2928b9b8bc71c938cb2b7200e48d3e","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-l5rcnr\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-l5rcnr\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622919087788032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"a7588ae59ddd79bdf62a22115b0a4decd0527f7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T12:31:13Z","started_at":"2026-10-12T12:31:13Z","completed_at":"2026-10-12T14:53:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:00:02.208144744Z","timing_metadata_checked_at":"2026-10-12T15:00:02.208144744Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:00:02.212164032Z","region_metadata_checked_at":"2026-10-12T15:00:02.212164032Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"30b9fe5eb4ae8b5747de324b1739662134db44fe9ac1454d8d1d66e98d8646db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622919087788032","occurred_at":"2026-10-12T12:31:13Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:41:55Z\n Ended: 2026-10-12T13:54:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7788032-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622525343305728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7473,"pr_state":"closed","pr_sha":"3c74b09813c6276369e5c733dcbe7c8ccbc680bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T12:29:39Z","started_at":"2026-10-12T12:29:39Z","completed_at":"2026-10-12T14:53:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:00:02.494351366Z","timing_metadata_checked_at":"2026-10-12T15:00:02.494351366Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:00:02.497937255Z","region_metadata_checked_at":"2026-10-12T15:00:02.497937255Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7471,"pr_state":"open","pr_sha":"1f96e5f6cfbb01c378a75805f2eaac811ab0914e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T12:06:11Z","started_at":"2026-10-12T12:06:11Z","completed_at":"2026-10-12T14:35:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T14:38:59.429550556Z","timing_metadata_checked_at":"2026-10-12T14:38:59.429550556Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T14:38:59.43223935Z","region_metadata_checked_at":"2026-10-12T14:38:59.43223935Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"75ae54f106342b191a9782550af2563b65c074c98f984315875cc054cc5b317f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"9542bbc82ccfd1f293ab2eb22d0e97d5eb5ed39e10bdfe00e2e3b03edd628b27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"c466496e418704ea254fc6bfcb21f24c936a9bb40e6d0c5ef2d2746c144dd4c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"External auth issuer validation rejects an invalid external auth issuer URL without leaving a child or changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"e91657699b49cccd6a0fd6048eb622153021ca47bf08d5dba8edcaa963371570","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109612109212946432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7467,"pr_state":"open","pr_sha":"395ae2305c3515cbbbe03da6d1c4c686fc44b301","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:48:16Z","started_at":"2026-10-12T11:48:16Z","completed_at":"2026-10-12T14:18:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T14:28:43.008713751Z","timing_metadata_checked_at":"2026-10-12T14:28:43.008713751Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T14:28:43.012709638Z","region_metadata_checked_at":"2026-10-12T14:28:43.012709638Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d6ad2c0d82b2213969019e29b55b2f8dfa58393d9ecf8bf0525d1da99ebcd2a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109612109212946432","occurred_at":"2026-10-12T11:48:16Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:04:34Z\n Ended: 2026-10-12T13:26:30Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2946432-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7472,"pr_state":"open","pr_sha":"bf2854016946a4de8b1934bb21d6e51c1684dd74","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:44:52Z","started_at":"2026-10-12T11:44:52Z","completed_at":"2026-10-12T14:30:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T14:38:59.628220673Z","timing_metadata_checked_at":"2026-10-12T14:38:59.628220673Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T14:38:59.633100661Z","region_metadata_checked_at":"2026-10-12T14:38:59.633100661Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"260600d2916d198d3d8597bd4dbe59fe6bfcdf8fb50b4a1f0db3b7ddeb5027ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"84a944cc4d17ddd0748f8f51deaf59c3bb580e3b149c031cb5f024447fa41bb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"External auth issuer validation rejects an invalid external auth issuer URL without leaving a child or changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"8534627d391dac834171b2a8af01b11ab2933bc0c3480b08334b36e382877080","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"alert","signature_id":"ace1503ea85ab8f2b4df98ae24ce5d1ca17bf7dd773d4068dfe89225c5ed9594","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T13:44:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T13:44:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T13:45:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-12T13:45:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/cluster-api has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"88b36fbeafa0e3fe8a22e53474e45652408f68a99769ff9374ef279dc917909f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Node pool create validation rejects a negative replica count without creating a node pool or changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"9145046ac6c3840e6b9548c6c630787c02b4f853e942b3e9d4136b67f4d15618","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d8e81683f310cfd380eec1bfc0de37cc0afdb111c53af29a24fca72bfb2507d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"alert","signature_id":"4885c3029699261783a90f8a5a1cf25063711e5d2be18ed2e9aec6344e30e22d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:30:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-9pjgh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-9pjgh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:25:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-mqbpc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-mqbpc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:25:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-jhk97\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-jhk97 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:34:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-b9n46\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-b9n46 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:34:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-cg97m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-cg97m has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-12T13:06:43Z\n Ended: 2026-10-12T13:25:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-kpf7b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-kpf7b has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-12T13:13:45Z\n Ended: 2026-10-12T13:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i\", pod=\"openshift-apiserver-6b4ff7ff95-9kn96\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i/openshift-apiserver-6b4ff7ff95-9kn96 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-12T13:15:42Z\n Ended: 2026-10-12T13:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-7cvh9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-7cvh9 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-12T13:15:42Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-8mskw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-8mskw has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-12T13:15:42Z\n Ended: 2026-10-12T13:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-j42hz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-j42hz has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-12T13:15:42Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-mkdz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-mkdz4 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-12T13:15:42Z\n Ended: 2026-10-12T13:36:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-5xwvh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-5xwvh has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-npkzv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-npkzv has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-m8dx4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-m8dx4 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-pnr6f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-pnr6f has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-gw6tj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-gw6tj has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-5x6tc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-5x6tc has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-6lcvg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-6lcvg has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i\", pod=\"openshift-apiserver-6b4ff7ff95-mksfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i/openshift-apiserver-6b4ff7ff95-mksfs has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-12T13:15:43Z\n Ended: 2026-10-12T13:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-bx6hr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-bx6hr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"dd9fe1932abe6b60a4bcbafeef8dfbacb718d2b591cffce10738d4f1c09d41b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"895d6dc730935dbb7c49cd2e90968b291d7469692edac24ed546ec1e8097382f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mb49f2vf2mck/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mb49f2vf2mck/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"e34b1b6ddac6837e32229c8b4d38efb3be157fd025efc3720ef69f736a53f04f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"edfa6c8fb88d3d6939b4f56ed76112bef69503e83c0057b7b8aff71355257791","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","failed_at":"e2e","signature_id":"f1b1b884321d5b2ac6d5d82d2bb343fddd2a8c66964ab52673d16c25eb8d5697","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7470,"pr_state":"open","pr_sha":"f3cccb6e50c8ef92b557158822920ea965959ba2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:44:03Z","started_at":"2026-10-12T11:44:03Z","completed_at":"2026-10-12T14:30:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T14:38:59.211490987Z","timing_metadata_checked_at":"2026-10-12T14:38:59.211490987Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T14:38:59.225280153Z","region_metadata_checked_at":"2026-10-12T14:38:59.225280153Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0078545e27ea4cd88398e5ce381727db7552803aade8d7d749a4fd93e11228e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"063dd9e174bcb9fb9688199fe98b04c49cfc034db89cd1cfc8a57eda24b3afdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"e2e","signature_id":"a902cb53b68f347d3389b746e7a2281dfe680f58c4b7f49f68ca93a281c00759","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22fbdb35a870cfc5f2c9e8846465e0b7f85e96d6e430537b6f1feed38d45976b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code KeyVaultAccessForbidden; detail message Unable to access key vault resource '\u003curl\u003e' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0ef1b979d6ad65d986f6622da7d5bbb8e6fd5c362c0b56576ff80e4f41045413","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"alert","signature_id":"ccfda79db149eab1089427f6b12af29fcb8d7b25163196bd92ebcab836f644a6","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:08:11Z\n Ended: 2026-10-12T13:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d015a5d-c4df-4754-bdd2-c1b853805f1f\"\n Description: Pod ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-12T13:09:11Z\n Ended: 2026-10-12T13:33:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d015a5d-c4df-4754-bdd2-c1b853805f1f\"\n Description: Pod ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"20b1925530ee430fa57a9c9693c4926c49bf7041aff44938dad6771af1c7a4a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"alert","signature_id":"59230409e21419e206546b3eed7180a2b2d9241870b6db3da89dea2764805770","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:06:14Z\n Ended: 2026-10-12T13:34:15Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T13:06:14Z\n Ended: 2026-10-12T13:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"3ba2b93eaa1249971a2e68c38563b4f4669a9f86c7bf11207d57dfcf26beb3a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:00:26Z\n Ended: 2026-10-12T13:09:30Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5598976-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"47962fce607304c10258194f36d50986b0e5f30ea3450ff2dade0492405d9c56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"alert","signature_id":"18f5e818c6420e89e3d88cbc04dfa007473088c4fb38a457467fa320874574d7","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:20:16Z\n Ended: 2026-10-12T13:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T13:21:13Z\n Ended: 2026-10-12T13:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6a19c1c8d990b519b33bf65ef9aaf55e62d6bf021c7bb188944b19af409c89a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"alert","signature_id":"eb4f11f09029df76c1cd0c8884283989b59eee33dde5e273ada7dc186fddbab8","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 119 time(s) (93 unknown, 26 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-phq7k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-phq7k has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-p9svn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-p9svn has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-sn7pp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-sn7pp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-g4rgv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-g4rgv has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-4c79f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-4c79f has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-5d87c549dd-xft98\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-5d87c549dd-xft98 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-8pxss\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-8pxss has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-wp75c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-wp75c has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-6d4b4c596d-kgm8k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-6d4b4c596d-kgm8k has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-5d87c549dd-jfjg4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-5d87c549dd-jfjg4 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-ptd4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-ptd4w has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-84sz2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-84sz2 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-j6427\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-j6427 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-n7mnp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-n7mnp has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-6d4b4c596d-bvqqj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-6d4b4c596d-bvqqj has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-jxwtv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-jxwtv has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-c8g6x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-c8g6x has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-5d87c549dd-nsvfd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-5d87c549dd-nsvfd has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-6d4b4c596d-n9gt9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-6d4b4c596d-n9gt9 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-d7rgr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-d7rgr has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-v2kpf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-v2kpf has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-wbdqf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-wbdqf has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-d52gv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-d52gv has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-6d4b4c596d-p5krx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-6d4b4c596d-p5krx has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-qklrl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-qklrl has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-956mt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-956mt has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-757rl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-757rl has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-4b6hz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-4b6hz has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-zckfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-zckfs has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:26:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"kube-controller-manager-866b6df4c4-wxk72\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/kube-controller-manager-866b6df4c4-wxk72 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-12T13:03:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p\", pod=\"azure-file-csi-driver-controller-5d87c549dd-pcdzb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fllch98do8t1mnl5upc3i6hqfnnp-d9e2s3z6q2u6v1p/azure-file-csi-driver-controller-5d87c549dd-pcdzb has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-12T13:04:13Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-7kcvb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-7kcvb has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-fb7bn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-fb7bn has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-96x7z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-96x7z has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-p2wsf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-p2wsf has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-kkqwz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-kkqwz has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-md6p4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-md6p4 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-lrll2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-lrll2 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-xmbjm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-xmbjm has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-phkhn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-phkhn has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-r9jwz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-r9jwz has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-kxwxf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-kxwxf has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-8p6bv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-8p6bv has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-7rq4z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-7rq4z has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-jc4df\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-jc4df has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-2c6lt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-2c6lt has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-7qsp9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-7qsp9 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:23:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-s5nkm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-s5nkm has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-mq9bs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-mq9bs has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-12T13:05:11Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-dq8cp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-dq8cp has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-g49bf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-g49bf has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-lqt44\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-lqt44 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-2krkg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-2krkg has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-mwzbm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-mwzbm has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-rsdjb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-rsdjb has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-8cq7w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-8cq7w has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-p8kv5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-p8kv5 has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-twjkp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-twjkp has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-vncn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-vncn4 has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-4b79p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-4b79p has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-8dv7s\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-8dv7s has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-44qv4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-44qv4 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-p428l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-p428l has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-zhslf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-zhslf has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-72sgc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-72sgc has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-sv54r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-sv54r has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-d9rsz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-d9rsz has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-br96b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-br96b has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-t8xgx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-t8xgx has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-8t44f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-8t44f has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-tf65g\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-tf65g has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-vs6dq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-vs6dq has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-8krt7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-8krt7 has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-btj2q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-btj2q has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-rf8gr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-rf8gr has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-cq958\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-cq958 has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-dzn28\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-dzn28 has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-km8hh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-km8hh has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-m6f6k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-m6f6k has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-c5bb7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-c5bb7 has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-ts6d9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-ts6d9 has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-bgwqx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-bgwqx has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-g9kxt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-g9kxt has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-lpthv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-lpthv has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-fdl8s\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-fdl8s has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-xs5wf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-xs5wf has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-75bdt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-75bdt has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:20:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v\", pod=\"ignition-server-proxy-7ddb9b85d9-vxr64\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8fn7evvpbjsrtq4vi1j4snca0v678-o5y8w5x8p9c0y1v/ignition-server-proxy-7ddb9b85d9-vxr64 has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w\", pod=\"azure-disk-csi-driver-controller-5b8cc5d6c8-6rgs4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8foj4rgg64afk1khume99dsthl10u-o7a9e0w4b8q9k1w/azure-disk-csi-driver-controller-5b8cc5d6c8-6rgs4 has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"control-plane-operator-59ffc49d8c-4svlv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/control-plane-operator-59ffc49d8c-4svlv has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:23:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-wtzg6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-wtzg6 has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-8vm5m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-8vm5m has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n State: Resolved\n Started: 2026-10-12T13:05:12Z\n Ended: 2026-10-12T13:27:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c\", pod=\"cluster-api-59c88c8d8f-7n5w9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td8g5jiacb863c3jitg24j7abmo6c7h-s4f2t4t5y0i3a4c/cluster-api-59c88c8d8f-7n5w9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"7bc2bdf75d2d074edb0794b9ded5e371fc060da7ccf06fe10eccfee1f228b393","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"802bb2dbdd57fe5ef19da0fd1eae5b7638bd344873f87774f47b7c7354940cf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"e2e","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","test_suite":"rp-api-compat-all/parallel","failure_text":"provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"9a5cf4e487ed3566a979bab33ac42244251a9869f2bc792611f047ef5068deb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","failed_at":"e2e","signature_id":"614ca0e1ea00ef1b4414e6ab8b3a11a8b2da5132a3da64d8b9faf88c1203affd","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x329d818e85e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7hcwvl89hlds\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\",\n \"name\": \"af485567-16d3-466b-8cef-6a0e9bce52fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:42:57.954228784Z\",\n \"endTime\": \"2026-10-12T13:11:58.34288805Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n correlation ID: 70b5741c-ea80-4b9a-9e33-ac7a70af1ce7\n client request ID: f176988a-7ecb-4ecf-8063-4ee5d27bb712\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x329d818e85e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7hcwvl89hlds\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\",\n \"name\": \"af485567-16d3-466b-8cef-6a0e9bce52fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:42:57.954228784Z\",\n \"endTime\": \"2026-10-12T13:11:58.34288805Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n correlation ID: 70b5741c-ea80-4b9a-9e33-ac7a70af1ce7\n client request ID: f176988a-7ecb-4ecf-8063-4ee5d27bb712\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","22fbdb35a870cfc5f2c9e8846465e0b7f85e96d6e430537b6f1feed38d45976b","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109603048882966528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"open","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:12:15Z","started_at":"2026-10-12T11:12:15Z","completed_at":"2026-10-12T13:28:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T13:35:27.10751083Z","timing_metadata_checked_at":"2026-10-12T13:35:27.10751083Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T13:35:27.112357915Z","region_metadata_checked_at":"2026-10-12T13:35:27.112357915Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"26fb32c6804eb13b19441bbb3933fd295ce7a41c46553c358411b5765290f118","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109603048882966528","occurred_at":"2026-10-12T11:12:15Z","failed_at":"e2e","signature_id":"bbe6ff38e69c56b0da8c72190329d55c3f58aff3ba835f793e6ee32876e960e6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7468,"pr_state":"open","pr_sha":"209c55bcf8a51457cd4564e430b8501d7e8956fa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:01:58Z","started_at":"2026-10-12T11:01:58Z","completed_at":"2026-10-12T13:40:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T13:46:27.181405615Z","timing_metadata_checked_at":"2026-10-12T13:46:27.181405615Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T13:46:27.184420707Z","region_metadata_checked_at":"2026-10-12T13:46:27.184420707Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"009bdf37847fa99db96f116e7d4a76e70a54089253e5587419ad88e6f9e05599","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T12:20:39Z\n Ended: 2026-10-12T12:36:40Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5767424-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"06a7cdd28c33739ac4f8f99c2ddc62afeb307ada85e80bee8f21d230dad57819","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","failed_at":"e2e","signature_id":"5b41a952bcf6d2d929fc1668bfb207665a25d88b8be5793b80fd4bb0cf28e575","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0xde424e4090\u003e: \n resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0xde424e4090\u003e: \n resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"48e35aa28e0d6cfd6e56a0cc8d2fc2540c7d8051ee8bed6e403f654ac64540d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","failed_at":"alert","signature_id":"a980a63c43707ea1bb98420a0dc8810e1d7f957a882e906bd2844388f2b4bbc5","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T13:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5767424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5767424/providers/microsoft.monitor/accounts/hcps-j5767424\", namespace=\"ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5prtf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T13:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5767424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5767424/providers/microsoft.monitor/accounts/hcps-j5767424\", namespace=\"ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5prtf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T13:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5767424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5767424/providers/microsoft.monitor/accounts/hcps-j5767424\", namespace=\"ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5prtf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-12T13:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5767424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5767424/providers/microsoft.monitor/accounts/hcps-j5767424\", namespace=\"ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5prtf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td7rmojjop6e5ni0mk3h0r2c95v0aqq-e8e5t4a2m8v7b5f/cluster-api has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600223830478848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7466,"pr_state":"closed","pr_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","final_merged_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T11:01:02Z","started_at":"2026-10-12T11:01:02Z","completed_at":"2026-10-12T13:00:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T13:03:14.810354699Z","timing_metadata_checked_at":"2026-10-12T13:03:14.810354699Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T13:03:14.813003995Z","region_metadata_checked_at":"2026-10-12T13:03:14.813003995Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600098458537984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7383,"pr_state":"open","pr_sha":"53e45f9fcccdb762505b423c56e38f3fce48c999","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T11:00:15Z","started_at":"2026-10-12T11:00:15Z","completed_at":"2026-10-12T13:47:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T13:56:59.139959245Z","timing_metadata_checked_at":"2026-10-12T13:56:59.139959245Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T13:56:59.15948219Z","region_metadata_checked_at":"2026-10-12T13:56:59.15948219Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6ff25388e1b48c63f917ce5ef91fb8aede2fcf70acd6948b35c740fb8359d2ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600098458537984","occurred_at":"2026-10-12T11:00:15Z","failed_at":"e2e","signature_id":"f0121469ca4f42ea3b4cc9ff22e2d69e6a7cdd34679fb94a79eb413022a0a715","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-shw9zg\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-shw9zg\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109599905499582464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"open","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T10:59:45Z","started_at":"2026-10-12T10:59:45Z","completed_at":"2026-10-12T13:34:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T13:46:26.768390695Z","timing_metadata_checked_at":"2026-10-12T13:46:26.768390695Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T13:46:26.772825184Z","region_metadata_checked_at":"2026-10-12T13:46:26.772825184Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6b0d33c79519b2c313dd22bbbd8be3153d6718dc40e809d47690f043fc7d7846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109599905499582464","occurred_at":"2026-10-12T10:59:45Z","failed_at":"e2e","signature_id":"01ff5d20cf309ae0ced3e300e16a31a96dd5d737827ad14c8718b21686301352","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-fbz6pthxmhdp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-fbz6pthxmhdp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7420,"pr_state":"closed","pr_sha":"4f30e2ce59cc606b3bf85418f97decf933428fa1","final_merged_sha":"4f30e2ce59cc606b3bf85418f97decf933428fa1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T10:14:00Z","started_at":"2026-10-12T10:14:00Z","completed_at":"2026-10-12T12:45:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T12:53:11.540720593Z","timing_metadata_checked_at":"2026-10-12T12:53:11.540720593Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T12:53:11.544006485Z","region_metadata_checked_at":"2026-10-12T12:53:11.544006485Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6a86b13fa8c0fe063525de10c97d73709331da16f442a7649e93ea59d01121dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","failed_at":"e2e","signature_id":"810897085d9afd12e972e9199e65575a09acca1a53e133ff2eb94e81c5444daf","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d26145f8800c52e5ffc5918d2f2e8c12677ef10fdb0eac407bd8dab6ebdbf77e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","failed_at":"e2e","signature_id":"670cbec864bb9c486cb9441591ed8f63fe3036fd2cb244750889987cfff959c9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-82f6cfwmh699'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x258297402990\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-82f6cfwmh699'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x258297402990\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1b9753d6df9613da46b4b9cfaf32730b876169734288bd96a4ec92c7f1a772fa","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04"}},{"row_id":"e7a0a19c00fdf6ca78a91874e39b689443911a255a04f9aa07bb85fee08fb635","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","failed_at":"e2e","signature_id":"6ca522238814ece9529f702529a6bcb452636d575084a8828e41559f858abbbd","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9da4ad2c21fc0e265207db4c5abcfdd0631935b407c3edcd41248ee669719f25","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1b9753d6df9613da46b4b9cfaf32730b876169734288bd96a4ec92c7f1a772fa","9da4ad2c21fc0e265207db4c5abcfdd0631935b407c3edcd41248ee669719f25","a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109584061419753472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7458,"pr_state":"closed","pr_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","final_merged_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T09:56:49Z","started_at":"2026-10-12T09:56:49Z","completed_at":"2026-10-12T12:19:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T12:21:58.046352029Z","timing_metadata_checked_at":"2026-10-12T12:21:58.046352029Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T12:21:58.049829119Z","region_metadata_checked_at":"2026-10-12T12:21:58.049829119Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"open","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T09:45:06Z","started_at":"2026-10-12T09:45:06Z","completed_at":"2026-10-12T10:48:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T11:08:35.679637968Z","timing_metadata_checked_at":"2026-10-12T11:08:35.679637968Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T11:08:35.68227916Z","region_metadata_checked_at":"2026-10-12T11:08:35.68227916Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"1cfb1f13061ada88076068dda6d43d7accf6e4cf31b22d1cf46a460cfbfd4772","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","failed_at":"alert","signature_id":"6e8b49a5caf85ad1a9ceb24783ca3a6237c65bb0981b2afa932bd94b35ce4433","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T10:37:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-vk9xs\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-vk9xs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T10:42:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-64b5859f7f-5glfr\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-64b5859f7f-5glfr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T10:42:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"fleet\", pod=\"fleet-controller-76c4b4c8bc-8gvkr\", region=\"westus3\", severity=\"warning\"\n Description: Pod fleet/fleet-controller-76c4b4c8bc-8gvkr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"b0b190a7bf43ac93d15da454e31046e3bba56c26d7f136053a2f4137214a2c23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","failed_at":"provision","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-12T10:35:56.708Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:57.601Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:35:57.906Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:35:57.907Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:58.041Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:40:59.966Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:40:59.967Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"21c5097072b27775aeb5bfa1e803499de79c8013","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T09:35:22Z","started_at":"2026-10-12T09:35:22Z","completed_at":"2026-10-12T10:40:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T10:47:57.215972117Z","timing_metadata_checked_at":"2026-10-12T10:47:57.215972117Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T10:47:57.218757508Z","region_metadata_checked_at":"2026-10-12T10:47:57.218757508Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"851cb782f5b2f0cbcb86bc4ea64efcdf5b19d371c680224f8e70e3d97c5a651f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","failed_at":"provision","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-12T10:30:06.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:07.671Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:08.275Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:10.285Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:35:10.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded"}},{"row_id":"c67046c1adf94b9cb43f2e44d75be13f1b719c6e53866a0aa81a08a1024eb6c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","failed_at":"alert","signature_id":"b224cee0c66252329215d260009da282cdb5e711becf7fce204c7d1cc07cca6f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000002-dskt8\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14292482-vmss000002-dskt8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000000-472fv\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14292482-vmss000000-472fv has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T10:31:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dmqmp\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-dmqmp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109576437055885312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7466,"pr_state":"closed","pr_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","final_merged_sha":"2ebef17f3dfd58b0d446bc018d6b85ab283aca0f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T09:26:31Z","started_at":"2026-10-12T09:26:31Z","completed_at":"2026-10-12T10:27:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T10:37:24.970362591Z","timing_metadata_checked_at":"2026-10-12T10:37:24.970362591Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T10:37:24.973201681Z","region_metadata_checked_at":"2026-10-12T10:37:24.973201681Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a630d2a19b5fa47b284f2c6f67828a78d1e44d81f39103bcda00a641b5078544","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109576437055885312","occurred_at":"2026-10-12T09:26:31Z","failed_at":"provision","signature_id":"5ac3f50227d6a7076813d8bbbe9de0e8c8876a26e6800117bb2654c1677cfb10","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","test_suite":"templatize-pipeline","failure_text":"time=2026-10-12T10:17:07.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:b0cbba7044d5142f8545758e5a69b7b35e7f8c6af596bc972e164d2b345978a7 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.589Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull namespace=acrpull\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull dryRun=false\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.976Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:22:16.134Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-12T10:22:16.135Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b3e3dde3ed6e86eca14ea52f55b0d41a64a9d455bb9f2e88e86dfa581de65743","failure_pattern":"failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b3e3dde3ed6e86eca14ea52f55b0d41a64a9d455bb9f2e88e86dfa581de65743"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7421/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109572353028526080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7421,"pr_state":"open","pr_sha":"ddfc51ee83bed5c35195b58811b527e6ee237bdc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T09:10:14Z","started_at":"2026-10-12T09:10:14Z","completed_at":"2026-10-12T11:21:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T11:39:44.606870615Z","timing_metadata_checked_at":"2026-10-12T11:39:44.606870615Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T11:39:44.610313405Z","region_metadata_checked_at":"2026-10-12T11:39:44.610313405Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109566058561015808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7219,"pr_state":"open","pr_sha":"0e1c991ca2d9b9776ba755f464d7ccdc27554ba6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T08:45:16Z","started_at":"2026-10-12T08:45:16Z","completed_at":"2026-10-12T11:06:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T11:08:35.651831654Z","timing_metadata_checked_at":"2026-10-12T11:08:35.651831654Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T11:08:35.654437746Z","region_metadata_checked_at":"2026-10-12T11:08:35.654437746Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"92a600342253de164c40be253d6ad15a7ac7ba1e5b2533ca3a7f526af82f073b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109566058561015808","occurred_at":"2026-10-12T08:45:16Z","failed_at":"alert","signature_id":"16da3c90424d3e20001b8e7c2d884bfc967a23a24761b0cf7cc40ac186996cdd","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T10:44:16Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1015808-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1015808/providers/microsoft.monitor/accounts/services-j1015808\", namespace=\"klusterlet-2td5olqiam9j0qubto977v59hbae6bo7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-m2h4p\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2td5olqiam9j0qubto977v59hbae6bo7 on cluster ci01-j1015808-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"open","pr_sha":"5aaf36ccd7ab21e7b010905f7dcae935e28e8561","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T08:35:17Z","started_at":"2026-10-12T08:35:17Z","completed_at":"2026-10-12T10:51:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T11:08:35.698768208Z","timing_metadata_checked_at":"2026-10-12T11:08:35.698768208Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T11:08:35.700873002Z","region_metadata_checked_at":"2026-10-12T11:08:35.700873002Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"605979734534315186b544549102ebaf17fd30db689e852fc63aa2404c861838","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","failed_at":"alert","signature_id":"cfbb3abf844cd53f28b64e3663605f410242ad10ab7e9f569183150881111e11","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T09:49:03Z\n Ended: 2026-10-12T10:13:58Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4hsb2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T09:49:03Z\n Ended: 2026-10-12T10:17:57Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4hsb2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"62a56701d6825cd4fdc5d7e82add80765e4b0d31ae7d8ceaed57fa08a1c99761","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","failed_at":"alert","signature_id":"e9c8652d1165ca174ad5201f1063a5d60e6d9e1e776a81a9de4becc5e6a83ae5","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T09:50:56Z\n Ended: 2026-10-12T10:17:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f3cafcc-c004-4fb5-b689-5b2c6b0de186\"\n Description: Pod ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-12T09:50:56Z\n Ended: 2026-10-12T10:17:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f3cafcc-c004-4fb5-b689-5b2c6b0de186\"\n Description: Pod ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"b39363f41866b6d3f83798ea890dc4fb06e790788b875f316ffc90eed05d2777","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","failed_at":"e2e","signature_id":"f06a1fb8a7fb87970f3aa02a8622a506c32fd712c86f231325a5e5fce4359407","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b4c7eddde1328d7ef9a967d1bbd9e544ff678448c8aea7bee0071d8ff0cf08ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","failed_at":"alert","signature_id":"c2c5cbd8dc969313ed2d41b08a27fef289acf32d3365fe9e6a45d7383b454d11","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T10:03:02Z\n Ended: 2026-10-12T10:13:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4hsb2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ff7fecbb14584da820a449bd9282536af600bbdb3133d63eb6af7c7322888325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","failed_at":"e2e","signature_id":"026e1543f757852d519080f003f4d7ebec18027795d5835ee8f594400d820c5e","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d2ba8a26f40\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmxkg7m275vm\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\",\n \"name\": \"06ab8b07-36db-4738-9b5d-4d16092f8301\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:26:40.450447729Z\",\n \"endTime\": \"2026-10-12T09:55:40.068785838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n correlation ID: c5b7b9a4-edc0-484e-9068-a103f50c2ab8\n client request ID: d0e69db4-dd9a-407e-93a4-3206c81ac5a6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d2ba8a26f40\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmxkg7m275vm\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\",\n \"name\": \"06ab8b07-36db-4738-9b5d-4d16092f8301\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:26:40.450447729Z\",\n \"endTime\": \"2026-10-12T09:55:40.068785838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n correlation ID: c5b7b9a4-edc0-484e-9068-a103f50c2ab8\n client request ID: d0e69db4-dd9a-407e-93a4-3206c81ac5a6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109555123024826368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"a7588ae59ddd79bdf62a22115b0a4decd0527f7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T08:00:48Z","started_at":"2026-10-12T08:00:48Z","completed_at":"2026-10-12T10:20:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T10:26:38.561319592Z","timing_metadata_checked_at":"2026-10-12T10:26:38.561319592Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T10:26:38.564234383Z","region_metadata_checked_at":"2026-10-12T10:26:38.564234383Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"031edb5c4cb9e07b742ea7c814d56fa9a3f188e4341b6a4444d1b3ce6678abb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109555123024826368","occurred_at":"2026-10-12T08:00:48Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T09:09:06Z\n Ended: 2026-10-12T09:25:06Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4826368-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109549126109630464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7237,"pr_state":"open","pr_sha":"19b4dbe3c46f37e6170d0d3479c8e819bc98e0f8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T07:37:59Z","started_at":"2026-10-12T07:37:59Z","completed_at":"2026-10-12T09:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T10:06:03.322925225Z","timing_metadata_checked_at":"2026-10-12T10:06:03.322925225Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T10:06:03.326053315Z","region_metadata_checked_at":"2026-10-12T10:06:03.326053315Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"318cae8fca1bb2fc81ba2be65458134da696891a384cf94a805668e3bde37bf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109549126109630464","occurred_at":"2026-10-12T07:37:59Z","failed_at":"e2e","signature_id":"ebfef50e4f33847feb7608db52e9f990adccd34b42505d1aca31667f15925d12","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7465/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109543184655716352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7465,"pr_state":"closed","pr_sha":"10e4605b99f1e025287ff56022a910c0df585302","final_merged_sha":"10e4605b99f1e025287ff56022a910c0df585302","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T07:14:23Z","started_at":"2026-10-12T07:14:23Z","completed_at":"2026-10-12T09:20:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:33:44.111774587Z","timing_metadata_checked_at":"2026-10-12T09:33:44.111774587Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:33:44.117323771Z","region_metadata_checked_at":"2026-10-12T09:33:44.117323771Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109538389626720256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"open","pr_sha":"10b61f39885decdd5db9284af597684c6252c532","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T06:55:20Z","started_at":"2026-10-12T06:55:20Z","completed_at":"2026-10-12T09:10:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:23:11.805089891Z","timing_metadata_checked_at":"2026-10-12T09:23:11.805089891Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:23:11.808039582Z","region_metadata_checked_at":"2026-10-12T09:23:11.808039582Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"76e6b6caa07643732af3a3e2a8bd3c6b6018e20084ee36d955a7a7ebb29af7bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109538389626720256","occurred_at":"2026-10-12T06:55:20Z","failed_at":"e2e","signature_id":"4673534f224bf242b8df8885a07c9c85c02ab3a626129d3db6b153550df27ba6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"4aee7aa6683a39a895cbcd1ca9b36322be5dd020","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T06:48:01Z","started_at":"2026-10-12T06:48:01Z","completed_at":"2026-10-12T09:10:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:23:11.645383365Z","timing_metadata_checked_at":"2026-10-12T09:23:11.645383365Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:23:11.648957355Z","region_metadata_checked_at":"2026-10-12T09:23:11.648957355Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8d9c10b4864a3dff50c6e973822ef0c2732d0c930b6eb9cb018a5097b915e5a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","failed_at":"e2e","signature_id":"de4fcc56ca6c7e3d2de039fd00e411d285c0c7d0026255cd09fc37bbc4e1d4c7","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:160]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2462d4d4a5a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n \"name\": \"466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:57:35.126724285Z\",\n \"endTime\": \"2026-10-12T08:26:42.188209535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n correlation ID: df9f6d3b-b689-4825-a253-151f9584e7e4\n client request ID: 5fd0c7fe-22bf-4239-aeb7-b1d20c9c430c\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:160]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2462d4d4a5a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n \"name\": \"466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:57:35.126724285Z\",\n \"endTime\": \"2026-10-12T08:26:42.188209535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n correlation ID: df9f6d3b-b689-4825-a253-151f9584e7e4\n client request ID: 5fd0c7fe-22bf-4239-aeb7-b1d20c9c430c","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp"}},{"row_id":"e944463a2ad8cb5658a950d4c3eeeb73412931c0971eb8f289828e5fcec07ad4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","failed_at":"e2e","signature_id":"9b6a19c4bbd8e0fee198397011d61bfa3837fe3c4ea1afea5bc1a3c2f92f5308","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528983438626816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7458,"pr_state":"closed","pr_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","final_merged_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T06:17:57Z","started_at":"2026-10-12T06:17:57Z","completed_at":"2026-10-12T08:37:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T08:52:19.739068573Z","timing_metadata_checked_at":"2026-10-12T08:52:19.739068573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T08:52:19.741992164Z","region_metadata_checked_at":"2026-10-12T08:52:19.741992164Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"259e83761619647c147bfb1d7f6d519d5c5270ba0c74c59767a1270ec6f2d63d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528983438626816","occurred_at":"2026-10-12T06:17:57Z","failed_at":"alert","signature_id":"9a0460e320537545b7df7ed4ea07d8af2e7eacb2bec27a23e38614a9b14396d9","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T08:19:01Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j8626816-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.13:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626816/providers/microsoft.monitor/accounts/services-j8626816\", namespace=\"klusterlet-2td3l5vvknp9tr2ukktok1ca6hp11kor\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8hz5j\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2td3l5vvknp9tr2ukktok1ca6hp11kor on cluster ci01-j8626816-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7424,"pr_state":"closed","pr_sha":"c154328295ceb9b8e8a45b91b824588993971141","final_merged_sha":"c154328295ceb9b8e8a45b91b824588993971141","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T06:17:02Z","started_at":"2026-10-12T06:17:02Z","completed_at":"2026-10-12T08:47:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:02:51.830058295Z","timing_metadata_checked_at":"2026-10-12T09:02:51.830058295Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:02:51.833015086Z","region_metadata_checked_at":"2026-10-12T09:02:51.833015086Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"7bcf4b851649f83aa4d17ceb81dc0419e0abf62bd2cfc598a8560d0114690e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","failed_at":"alert","signature_id":"7f1c703708a0c73e567c797b757f2be0ee02cc20a38341bda264d7aaee221d0e","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T08:22:04Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j7802496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7802496/providers/microsoft.monitor/accounts/services-j7802496\", namespace=\"klusterlet-2td3ln668dv56op2785f7bjdpr831gpj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qmpf7\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2td3ln668dv56op2785f7bjdpr831gpj on cluster ci01-j7802496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"a1a2fd8f2e5210f14f80cbc27a09216095b19a12c30f930a14046971a576040a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","failed_at":"alert","signature_id":"36fc6cf9b7b57226fae3978e4f0d34c96bf304895ef33b2f585810930ebddbca","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T08:05:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7802496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7802496/providers/microsoft.monitor/accounts/hcps-j7802496\", namespace=\"ocm-arohcpci01-2td3lsap1436jgm959cn57jh1risiuj3-f9r1b8n1e1c4l7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qmpf7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td3lsap1436jgm959cn57jh1risiuj3-f9r1b8n1e1c4l7y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T08:12:25Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7802496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7802496/providers/microsoft.monitor/accounts/hcps-j7802496\", namespace=\"ocm-arohcpci01-2td3lsap1436jgm959cn57jh1risiuj3-f9r1b8n1e1c4l7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qmpf7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2td3lsap1436jgm959cn57jh1risiuj3-f9r1b8n1e1c4l7y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109522115974664192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7097,"pr_state":"open","pr_sha":"c696b67b646254630861f0200030228642b0476e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-12T05:50:40Z","started_at":"2026-10-12T05:50:40Z","completed_at":"2026-10-12T07:58:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T08:09:51.085379597Z","timing_metadata_checked_at":"2026-10-12T08:09:51.085379597Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T08:09:51.090206082Z","region_metadata_checked_at":"2026-10-12T08:09:51.090206082Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109519598771507200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7430,"pr_state":"open","pr_sha":"601ab018f4792fbaa64ebda154bd5b583691067f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T05:40:39Z","started_at":"2026-10-12T05:40:39Z","completed_at":"2026-10-12T08:15:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T08:31:11.956861716Z","timing_metadata_checked_at":"2026-10-12T08:31:11.956861716Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T08:31:11.960035807Z","region_metadata_checked_at":"2026-10-12T08:31:11.960035807Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8a57278ee17cd045febcac5d6000ff8104409608fbf0930cecae43f87ae87511","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109519598771507200","occurred_at":"2026-10-12T05:40:39Z","failed_at":"alert","signature_id":"6f8e2d109f354a776a1acb2c7fa60149b6a8d7d0329aa7b37c469a8320c9fe0f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T06:50:18Z\n Ended: 2026-10-12T07:11:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1507200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td30fa9ubf174gpmmj5sp08qe3vi0pk-z3e0o7o8p5c7d6x\", pod=\"ovnkube-control-plane-758b9b4b46-kgsrv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2td30fa9ubf174gpmmj5sp08qe3vi0pk-z3e0o7o8p5c7d6x/ovnkube-control-plane-758b9b4b46-kgsrv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109474078921330688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"open","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-12T02:39:47Z","started_at":"2026-10-12T02:39:47Z","completed_at":"2026-10-12T04:49:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T05:00:35.397223869Z","timing_metadata_checked_at":"2026-10-12T05:00:35.397223869Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T05:00:35.405164144Z","region_metadata_checked_at":"2026-10-12T05:00:35.405164144Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2523d9ce836f30983b66b3b9b956cf0d7ab966ad003f5959f7236bafffb27d95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109474078921330688","occurred_at":"2026-10-12T02:39:47Z","failed_at":"e2e","signature_id":"7e05f12cb3af3d2df3800593d5e4d70e1bcc0ddd1495a395607587d40f14bb85","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc252182d240\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c847fqslv6ss\" to finish creating: GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\",\n \"name\": \"de9482bc-af9d-45d5-a419-3811860ecab1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:33:37.063419631Z\",\n \"endTime\": \"2026-10-12T04:02:41.894551349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n correlation ID: 5048e7cb-ed4b-40b9-bfb8-b714b05f3bec\n client request ID: a436b4d6-2bc5-40f5-a8b1-84635292bdaa\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc252182d240\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c847fqslv6ss\" to finish creating: GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\",\n \"name\": \"de9482bc-af9d-45d5-a419-3811860ecab1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:33:37.063419631Z\",\n \"endTime\": \"2026-10-12T04:02:41.894551349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n correlation ID: 5048e7cb-ed4b-40b9-bfb8-b714b05f3bec\n client request ID: a436b4d6-2bc5-40f5-a8b1-84635292bdaa","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109460696247832576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7424,"pr_state":"closed","pr_sha":"c154328295ceb9b8e8a45b91b824588993971141","final_merged_sha":"c154328295ceb9b8e8a45b91b824588993971141","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T01:46:36Z","started_at":"2026-10-12T01:46:36Z","completed_at":"2026-10-12T03:59:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T04:18:30.715711468Z","timing_metadata_checked_at":"2026-10-12T04:18:30.715711468Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T04:18:30.719207955Z","region_metadata_checked_at":"2026-10-12T04:18:30.719207955Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5aaf41ae3649a2a089ffc4560ca9e1c0dc932a71d7407c7e7835c16007571e4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109460696247832576","occurred_at":"2026-10-12T01:46:36Z","failed_at":"e2e","signature_id":"8363386ebe99557c4d0a5f2ab5fc61129ce9cc8c82392c35b5213d30f26dde65","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":16,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":10,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109705414206558208","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T17:59:01Z","started_at":"2026-10-12T17:59:01Z","completed_at":"2026-10-12T20:42:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T20:57:46.812146935Z","timing_metadata_checked_at":"2026-10-12T20:57:46.812146935Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T20:57:46.815175625Z","region_metadata_checked_at":"2026-10-12T20:57:46.815175625Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109679585040797696","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T16:16:23Z","started_at":"2026-10-12T16:16:23Z","completed_at":"2026-10-12T18:52:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T19:02:28.946417183Z","timing_metadata_checked_at":"2026-10-12T19:02:28.946417183Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T19:02:28.949234775Z","region_metadata_checked_at":"2026-10-12T19:02:28.949234775Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109679208295829504","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T16:14:51Z","started_at":"2026-10-12T16:14:51Z","completed_at":"2026-10-12T16:43:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T16:45:51.850644114Z","timing_metadata_checked_at":"2026-10-12T16:45:51.850644114Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-12T16:45:51.853462405Z","region_metadata_checked_at":"2026-10-12T17:48:19.562373093Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a139fdbeef57906ca302064759256ce1243fd2f744d531dc2874cb671ebc3996","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109679208295829504","occurred_at":"2026-10-12T16:14:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109631847959891968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T13:06:42Z","started_at":"2026-10-12T13:06:42Z","completed_at":"2026-10-12T16:03:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T16:14:03.709642277Z","timing_metadata_checked_at":"2026-10-12T16:14:03.709642277Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T16:14:03.712562968Z","region_metadata_checked_at":"2026-10-12T16:14:03.712562968Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109630653069791232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T13:01:57Z","started_at":"2026-10-12T13:01:57Z","completed_at":"2026-10-12T15:48:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T15:53:14.747428777Z","timing_metadata_checked_at":"2026-10-12T15:53:14.747428777Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T15:53:14.749923069Z","region_metadata_checked_at":"2026-10-12T15:53:14.749923069Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f981d9f42ccd38dc564c619d97e19f0445d56ea08cf30060f5f70187156b55fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109630653069791232","occurred_at":"2026-10-12T13:01:57Z","failed_at":"e2e","signature_id":"625fe3d4a7bcc05f82b6be9c1b459cb3509760d899d8c1e3dcb5cf92e4d63ae3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109577566711975936","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T09:31:00Z","started_at":"2026-10-12T09:31:00Z","completed_at":"2026-10-12T12:01:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T12:21:58.158901406Z","timing_metadata_checked_at":"2026-10-12T12:21:58.158901406Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T12:21:58.16462439Z","region_metadata_checked_at":"2026-10-12T12:21:58.16462439Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109572070579900416","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T09:09:10Z","started_at":"2026-10-12T09:09:10Z","completed_at":"2026-10-12T11:36:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T11:50:00.753990379Z","timing_metadata_checked_at":"2026-10-12T11:50:00.753990379Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T11:50:00.757442969Z","region_metadata_checked_at":"2026-10-12T11:50:00.757442969Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109537039568343040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T06:49:58Z","started_at":"2026-10-12T06:49:58Z","completed_at":"2026-10-12T09:34:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:55:05.551789406Z","timing_metadata_checked_at":"2026-10-12T09:55:05.551789406Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:55:05.555819493Z","region_metadata_checked_at":"2026-10-12T09:55:05.555819493Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109535659495526400","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T06:44:29Z","started_at":"2026-10-12T06:44:29Z","completed_at":"2026-10-12T09:28:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:44:22.338445216Z","timing_metadata_checked_at":"2026-10-12T09:44:22.338445216Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:44:22.344644896Z","region_metadata_checked_at":"2026-10-12T09:44:22.344644896Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109535177171537920","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T06:42:34Z","started_at":"2026-10-12T06:42:34Z","completed_at":"2026-10-12T09:18:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T09:33:44.131929926Z","timing_metadata_checked_at":"2026-10-12T09:33:44.131929926Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T09:33:44.134746918Z","region_metadata_checked_at":"2026-10-12T09:33:44.134746918Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f806763e0716b0134aa5370673c3781e6b2ffb592060a3343d5b511acb72a33b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109535177171537920","occurred_at":"2026-10-12T06:42:34Z","failed_at":"e2e","signature_id":"fd4fc621d01f385247f810320f7e86a4ebf9edde1b95ddd444f5031ee94b4826","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109494571607003136","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T04:00:27Z","started_at":"2026-10-12T04:00:27Z","completed_at":"2026-10-12T06:24:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T06:35:05.570199216Z","timing_metadata_checked_at":"2026-10-12T06:35:05.570199216Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T06:35:05.573663205Z","region_metadata_checked_at":"2026-10-12T06:35:05.573663205Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109485667833614336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T03:25:50Z","started_at":"2026-10-12T03:25:50Z","completed_at":"2026-10-12T05:48:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T06:03:31.48910361Z","timing_metadata_checked_at":"2026-10-12T06:03:31.48910361Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T06:03:31.492824399Z","region_metadata_checked_at":"2026-10-12T06:03:31.492824399Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109477498252169216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T02:53:22Z","started_at":"2026-10-12T02:53:22Z","completed_at":"2026-10-12T03:08:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T03:25:50.355640783Z","timing_metadata_checked_at":"2026-10-12T03:25:50.355640783Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-12T03:25:50.360189169Z","region_metadata_checked_at":"2026-10-12T04:29:29.172730072Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e17ff41b23668c1a8b20670a0cfcf03fc8c4d6cdf8c1f14756a4951c4c372084","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109477498252169216","occurred_at":"2026-10-12T02:53:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2109464085383876608","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T02:00:04Z","started_at":"2026-10-12T02:00:04Z","completed_at":"2026-10-12T04:23:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T04:40:24.944935402Z","timing_metadata_checked_at":"2026-10-12T04:40:24.944935402Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T04:40:24.958057064Z","region_metadata_checked_at":"2026-10-12T04:40:24.958057064Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109448556992532480","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T00:58:22Z","started_at":"2026-10-12T00:58:22Z","completed_at":"2026-10-12T03:18:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T03:36:05.078228159Z","timing_metadata_checked_at":"2026-10-12T03:36:05.078228159Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T03:36:05.082076147Z","region_metadata_checked_at":"2026-10-12T03:36:05.082076147Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3a9d9a760ed5fd7497b278bdf77abef8fd56364a215b22a99078c1c37456c481","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109448556992532480","occurred_at":"2026-10-12T00:58:22Z","failed_at":"e2e","signature_id":"92ca5810cc30b4aac8e0fae2571490fb634c25c0c1d6fd8a60f745c71eb4e4ce","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc79feec4060\u003e: \n resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc79feec4060\u003e: \n resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8effde2aab4f5beaedda3d8b16dda29c381b8de0bb2c433491a2eb475fa67fb2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8effde2aab4f5beaedda3d8b16dda29c381b8de0bb2c433491a2eb475fa67fb2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-12T00:05:13Z","started_at":"2026-10-12T00:05:13Z","completed_at":"2026-10-12T02:36:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T02:54:27.243105933Z","timing_metadata_checked_at":"2026-10-12T02:54:27.243105933Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T02:54:27.247662019Z","region_metadata_checked_at":"2026-10-12T02:54:27.247662019Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"540badd3de3c1dc5e7e52b5161ebdff42cfcbcbe625c76926de63a488d0dc9bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","failed_at":"e2e","signature_id":"ca3816013b61a6551e43b1071dcaa2fa8f2e2f088ce6dca69491ec625e587d83","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87227a922c90e8b74b9cd5f3e1315585fd49c7030b59796956bc7a208ed9e9d1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c0911be3adbb547c3bb4ca59e0dadf1c9810ec5d7e4eb4689ff0230d02b5a89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","failed_at":"e2e","signature_id":"a33be23ec9f3ec13df877ffe26bd388352f59b753cee7526ea22101141d231fd","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87227a922c90e8b74b9cd5f3e1315585fd49c7030b59796956bc7a208ed9e9d1","f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":1,"failed_runs":0,"runs_with_occurrences":0,"runs_with_failure_pattern_matches":0,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2109464085413236736","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-12T02:00:04Z","started_at":"2026-10-12T02:00:04Z","completed_at":"2026-10-12T03:57:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T04:08:06.71561409Z","timing_metadata_checked_at":"2026-10-12T04:08:06.71561409Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T04:08:06.71914798Z","region_metadata_checked_at":"2026-10-12T04:08:06.71914798Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}