{"meta":{"date":"2026-09-18","timezone":"UTC","generated_at":"2026-10-15T00:56:21Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":84,"failed_runs":59,"runs_with_occurrences":59,"runs_with_failure_pattern_matches":49,"runs_with_unmatched_occurrences":10,"runs_without_occurrence_rows":25,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7017,"pr_state":"closed","pr_sha":"55788c289a67f3ac476a0ae85bf3de8260953664","final_merged_sha":"f177240b9d6b4d251ea43939d1ec48ab5b62d5f0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T23:03:02Z","started_at":"2026-09-18T23:03:02Z","completed_at":"2026-09-19T00:54:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.764675578Z","timing_metadata_checked_at":"2026-09-19T03:30:43.764675578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.768173891Z","region_metadata_checked_at":"2026-09-19T03:30:43.768173891Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"dd6353aebccaeca7eaadd48d97bcda89c5d2979821c82bceb1cfd6aa862e8e1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","occurred_at":"2026-09-18T23:03:02Z","failed_at":"alert","signature_id":"86c45d947e908dd24da56e029e4fb3fb50ede5048c717aadd07439573c0a5e5a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:47:51Z\n Ended: 2026-09-19T00:19:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8441088-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8441088-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7012,"pr_state":"closed","pr_sha":"2069fe0ee5d2249364832dbb2c3166d5a85d96e9","final_merged_sha":"2069fe0ee5d2249364832dbb2c3166d5a85d96e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T22:44:47Z","started_at":"2026-09-18T22:44:47Z","completed_at":"2026-09-19T00:36:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.136011775Z","timing_metadata_checked_at":"2026-09-19T03:30:43.136011775Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.139157486Z","region_metadata_checked_at":"2026-09-19T03:30:43.139157486Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"139b92f24a913d47aa240a4e218bdc9e4dbfef5ab826c9b8d524e6c859656832","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"a46151e80a37451e13b91d43941493c16950cf32997bf773ab888a8c5db06db6","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"579e01f42e94dee6a803ea631eada6ef71b7974e00977d9678afb887f3b1f56d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"7dead5f17d20c600ef9a15a67c9915db561b768a43db7c68ada9d07b7b90dfc6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:17:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:18:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"99ae21d96735666007952a512d8240b0d5951d0f234954f6074bb56dd7ba272f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"e2e","signature_id":"2c332da578489a812e0144aacf458767c886ed8769b126097a13ac0573609822","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d503582cd5359ec3a787862a0d3ae2209b75489992f83d22ae2e8e10e463450b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"61a9d86a254d9ba501b68a3f3cb4e321b30d8b314bb25ec5b72e60af5be20fd0","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:30Z\n Ended: 2026-09-19T00:23:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"e18a02cc81ba5f8eb087e5af03c24ecfeee3031e86b210b833a22ae3272e3914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"b0cb1249b73e4e17694cee1167aadd3f0fe5951d24186dea9efdb1984ff09dea","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:52:29Z\n Ended: 2026-09-19T00:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:53:28Z\n Ended: 2026-09-19T00:17:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101068742900322304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T21:59:58Z","started_at":"2026-09-18T21:59:58Z","completed_at":"2026-09-18T22:08:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.929506818Z","timing_metadata_checked_at":"2026-09-19T03:30:42.929506818Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-19T03:30:42.933952334Z","region_metadata_checked_at":"2026-09-19T04:34:06.858678691Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"414fd554e1a3395cbd82b855d7b7d647f94ff634a4ae2d270b726ebc1dfd13f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101068742900322304","occurred_at":"2026-09-18T21:59:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101065688771727360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"e7d302ddca8bb69f3f50165f6260d152b11bef67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:47:50Z","started_at":"2026-09-18T21:47:50Z","completed_at":"2026-09-18T21:56:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:39.889619581Z","timing_metadata_checked_at":"2026-09-19T03:30:39.889619581Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-19T03:30:39.894277798Z","region_metadata_checked_at":"2026-09-19T04:34:06.766254951Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f60224742cddf98497cedc0676c1042f2d25ee70b80e43834ed07fd17d181fc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101065688771727360","occurred_at":"2026-09-18T21:47:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"eae56fbadc2e1af941a3cecdb38bd7071e123b2c","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:36:30Z","started_at":"2026-09-18T21:36:30Z","completed_at":"2026-09-18T21:57:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:38.738880258Z","timing_metadata_checked_at":"2026-09-19T03:30:38.738880258Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:38.743353974Z","region_metadata_checked_at":"2026-09-19T03:30:38.743353974Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c3796091f6a6408c84986b62fb5fbfadcdf976575bddfe72195f2c827435c058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","occurred_at":"2026-09-18T21:36:30Z","failed_at":"provision","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-18T21:50:35.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T21:51:25.579Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"name\\\": \\\"04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'aad2e336-6c43-473c-997c-4a60a482583a'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4937088/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26091821etad3\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: b8850252-9aac-448f-9581-e29b2b8a10a8.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"open","pr_sha":"96c7afd52028f383231ff4e814218fb4e6b898b6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:22:20Z","started_at":"2026-09-18T21:22:20Z","completed_at":"2026-09-18T22:06:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.029064802Z","timing_metadata_checked_at":"2026-09-19T03:30:33.029064802Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.032546015Z","region_metadata_checked_at":"2026-09-19T03:30:33.032546015Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fd8055da033db00b4f97ef1c80a305e861fad1ca6a1da0262b2e7f9c777833d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","occurred_at":"2026-09-18T21:22:20Z","failed_at":"provision","signature_id":"b8972dd0ff9a3ca0c20207306d152fa4afd8e4eb6a37b8d1c512cde87ef5ae3e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-09-18T21:55:47.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-6 clientId:9c78a811-9c85-4eb0-9c84-d0843866c888 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj1086464.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj1086464.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Enabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f databaseName:arohcpci01-rp-j1086464 databaseUrl:https://arohcpci01-rp-j1086464.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-vh2hkbkg/pipeline@sha256:ec4f5311c601606f0bdcb4316ea9dff923b912c8e13a93addd5ac2c23dd9e86c replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:475m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-12 clientId:50d052a6-7aec-48b9-b43e-b8692a8f0ead clientIdSecretName: principalId:a76148d4-cb11-4b93-9363-54ba8239b0b1 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-vh2hkbkg/pipeline:pull workloadIdentityClientId:09708096-ca85-4992-8eb5-667a9b9d3fce workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:48.698Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:49.102Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"closed","pr_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","final_merged_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T21:00:47Z","started_at":"2026-09-18T21:00:47Z","completed_at":"2026-09-18T23:02:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.49756873Z","timing_metadata_checked_at":"2026-09-19T03:30:33.49756873Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.503477052Z","region_metadata_checked_at":"2026-09-19T03:30:33.503477052Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"25dae7a2a8aa574e5cc36a74517384b48ea9b23b21c576e82f1ba3cd82d07894","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","failed_at":"e2e","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"949106af16941bfb6470bdea16b80d0f435a326819d0ac895aa85186d1332c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","failed_at":"e2e","signature_id":"6566438295d27151c619c08c959579eb7709fb58c49e3c34c87dd1ec1d1e808b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053931038707712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T21:00:47Z","started_at":"2026-09-18T21:00:47Z","completed_at":"2026-09-18T23:12:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.319667784Z","timing_metadata_checked_at":"2026-09-19T03:30:42.319667784Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.323153197Z","region_metadata_checked_at":"2026-09-19T03:30:42.323153197Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7018,"pr_state":"open","pr_sha":"5c4e7b27f7d76d1aac25b9e8c9d004c3a38882f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T20:43:02Z","started_at":"2026-09-18T20:43:02Z","completed_at":"2026-09-18T22:38:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.902194082Z","timing_metadata_checked_at":"2026-09-19T03:30:43.902194082Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.905842595Z","region_metadata_checked_at":"2026-09-19T03:30:43.905842595Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0b55226030c8b821323f6ec09b006063aa5cd9671cb923b54989f1746921039e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","occurred_at":"2026-09-18T20:43:02Z","failed_at":"alert","signature_id":"900336073fa1807e3fa5eb270059d6e459df8256995e512e7d9ae978fff82f8c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T22:08:21Z\n Ended: 2026-09-18T22:18:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4790528-svc\", component=\"frontend\", method=\"delete\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4790528-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:52:27Z","started_at":"2026-09-18T19:52:27Z","completed_at":"2026-09-18T21:47:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:38.280963169Z","timing_metadata_checked_at":"2026-09-19T03:30:38.280963169Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:38.284919484Z","region_metadata_checked_at":"2026-09-19T03:30:38.284919484Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"07cd0904674d1540cc7d921fe4ddc5469c54ab0f3616558dfbe48b81bbce5f6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","occurred_at":"2026-09-18T19:52:27Z","failed_at":"alert","signature_id":"f8ce7b6dbb00b186dd3690b8bd0cde21b1999ed29a53641a6363be8c9bc8013e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T21:03:41Z\n Ended: 2026-09-18T21:09:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6750464-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6750464-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"e7d302ddca8bb69f3f50165f6260d152b11bef67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T19:39:52Z","started_at":"2026-09-18T19:39:52Z","completed_at":"2026-09-18T21:45:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:39.761505011Z","timing_metadata_checked_at":"2026-09-19T03:30:39.761505011Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:39.765341625Z","region_metadata_checked_at":"2026-09-19T03:30:39.765341625Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"155e4bac73019064e828e68620a1f1e6bb58c9e92c4ecfcb44479894745088e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","occurred_at":"2026-09-18T19:39:52Z","failed_at":"e2e","signature_id":"50e9ecb754e15646a314b5b99f973f998a45c92fb4fac3293bbdba2c5f6a2e7e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101024271722614784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T19:03:16Z","started_at":"2026-09-18T19:03:16Z","completed_at":"2026-09-18T21:03:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:40.405968472Z","timing_metadata_checked_at":"2026-09-19T03:30:40.405968472Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:40.409069784Z","region_metadata_checked_at":"2026-09-19T03:30:40.409069784Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023786441641984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T19:01:20Z","started_at":"2026-09-18T19:01:20Z","completed_at":"2026-09-18T21:18:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:34.081150382Z","timing_metadata_checked_at":"2026-09-19T03:30:34.081150382Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:34.085552198Z","region_metadata_checked_at":"2026-09-19T03:30:34.085552198Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:01:12Z","started_at":"2026-09-18T19:01:12Z","completed_at":"2026-09-18T20:12:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.149154859Z","timing_metadata_checked_at":"2026-09-19T03:30:42.149154859Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.152896373Z","region_metadata_checked_at":"2026-09-19T03:30:42.152896373Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1a1376dd0fb95d0eb56f649393a3cb9cf0da57657aae154c8d3840d96661bee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","occurred_at":"2026-09-18T19:01:12Z","failed_at":"provision","signature_id":"30e287a059c7694cef227767592b23353845f4281b98b8c7ff98febc06270f17","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-18T20:01:01.932Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 values=\"map[enabled:true image:map[digest:sha256:5177425b194b5d289664c62bedbdc95a42605bbfcde48fc7ddbb31bee0194b90 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.574Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 namespace=monitoring\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 dryRun=false\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.827Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:01:10Z","started_at":"2026-09-18T19:01:10Z","completed_at":"2026-09-18T21:16:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.669254465Z","timing_metadata_checked_at":"2026-09-19T03:30:42.669254465Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.672341176Z","region_metadata_checked_at":"2026-09-19T03:30:42.672341176Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"51d08b7a9ce6eb284dd77abd1ad96549aa4cd646cf58c08579dd431de0cb4b96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","failed_at":"alert","signature_id":"4beb77cb7f662a3ba0c82df6677c24959b741d146512bce1d7d72b33bd6e9a0e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T20:11:59Z\n Ended: 2026-09-18T20:26:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6109056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s\", pod=\"router-8444b6f56d-w6qnw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s/router-8444b6f56d-w6qnw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"59b46d6e51f79fb2f2cef722ec2d3bcef1e24821ac34f57d76534720f970aae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","failed_at":"e2e","signature_id":"c0b449ac08413f5f7083190089ef0d839099fa2699d64d1c9f2f8d26abbef529","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"closed","pr_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","final_merged_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T18:24:36Z","started_at":"2026-09-18T18:24:36Z","completed_at":"2026-09-18T20:20:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.31576216Z","timing_metadata_checked_at":"2026-09-19T03:30:33.31576216Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.327947105Z","region_metadata_checked_at":"2026-09-19T03:30:33.327947105Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c325b1238b2a4925b518747db89630c9a558c69c2b0b732e3b1da306fbbc469c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","occurred_at":"2026-09-18T18:24:36Z","failed_at":"e2e","signature_id":"96d0afdee41d99182089b8922944bd77cbf90a6f0b4ca2fe3e238fbfdaae440e","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7017,"pr_state":"closed","pr_sha":"55788c289a67f3ac476a0ae85bf3de8260953664","final_merged_sha":"f177240b9d6b4d251ea43939d1ec48ab5b62d5f0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T18:23:12Z","started_at":"2026-09-18T18:23:12Z","completed_at":"2026-09-18T18:56:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.598267568Z","timing_metadata_checked_at":"2026-09-19T03:30:43.598267568Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.603919489Z","region_metadata_checked_at":"2026-09-19T03:30:43.603919489Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c9cc4d15c1da0be79b46d0992c58977c3ef41fb98246c274c3cbea10aa1f2fd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","occurred_at":"2026-09-18T18:23:12Z","failed_at":"provision","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","test_suite":"step graph","failure_text":"time=2026-09-18T18:49:43.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n Kind: ARM\\n Template: templates/svc-infra-lookup.bicep\\n Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-18T18:49:44.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-18T18:49:45.666Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6366720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd\ntime=2026-09-18T18:50:16.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6366720-svc/providers/Microsoft.Resources/deployments/299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd/operationStatuses/08584118495004368821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:36:50Z","started_at":"2026-09-18T16:36:50Z","completed_at":"2026-09-18T18:38:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.48515329Z","timing_metadata_checked_at":"2026-09-19T03:30:42.48515329Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.488768903Z","region_metadata_checked_at":"2026-09-19T03:30:42.488768903Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2d03db804d08a5f94e0f525a3a7e2944810909b2021943645284f702064c910b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"adf91e76f1e8a11039895567a05fb0e5c3f1d56a9565f8cf0e71b7f1f6e5d0ce","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:37:21Z\n Ended: 2026-09-18T18:05:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9556480-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress\", pod=\"router-8495fb9c6b-n57p2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress/router-8495fb9c6b-n57p2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"499a269e139f43071276772a1c9a391a3bcc57585f049d433ae1cd506717be87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"e2e","signature_id":"99f885a8a863b5de08d9591137eee29d3c46e0095bea1b9e8046d833c161bfec","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"655707720d06a4babc91b86cb1ae99605111f9dd203a1eb99acc6c36f5dadcca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"1522dde0b9bb5d85e613b7efec8820e9ced25e3c11ff2b7a1c60a5642910649a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:56:41Z\n Ended: 2026-09-18T18:02:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9556480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"90fa4d791a743565dcbc1e61511ee86c5e89d4a029763e85151f644a46563bcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"f6e1d34656f1ce37365020fae4967ac3b4ae64788fb876cb79e6b050b731b2fe","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T18:04:41Z\n Ended: 2026-09-18T18:24:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9556480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100986377096138752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6805,"pr_state":"closed","pr_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","final_merged_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T16:17:54Z","started_at":"2026-09-18T16:17:54Z","completed_at":"2026-09-18T18:36:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:34.879501226Z","timing_metadata_checked_at":"2026-09-19T03:30:34.879501226Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:34.88329454Z","region_metadata_checked_at":"2026-09-19T03:30:34.88329454Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:10:25Z","started_at":"2026-09-18T16:10:25Z","completed_at":"2026-09-18T18:25:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.93947626Z","timing_metadata_checked_at":"2026-09-19T03:30:33.93947626Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.947493889Z","region_metadata_checked_at":"2026-09-19T03:30:33.947493889Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6fd9d882d77dd6c95fcad26bf18ad303ee02e15867f25ce324525bcdeb08b73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","occurred_at":"2026-09-18T16:10:25Z","failed_at":"alert","signature_id":"f7df40a156b9e56e6a5ddf150c7ac6e35049cd21802df2d33f93003f72173db8","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:42:43Z\n Ended: 2026-09-18T17:48:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3570304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3570304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:01:55Z","started_at":"2026-09-18T16:01:55Z","completed_at":"2026-09-18T18:45:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:41.705962035Z","timing_metadata_checked_at":"2026-09-19T03:30:41.705962035Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:41.710943154Z","region_metadata_checked_at":"2026-09-19T03:30:41.710943154Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b78d7fc419a09e5f4bd131af5dfb921f08e79bbc69566ea089cccfc2d63a07c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","occurred_at":"2026-09-18T16:01:55Z","failed_at":"alert","signature_id":"db6a9f6935fcb92fd2604ae71dc7832d2001b5cc7b8edb8be0b3faebabe391c0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:59:03Z\n Ended: 2026-09-18T18:05:04Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2544640-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2544640-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7014/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100985423357546496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7014,"pr_state":"closed","pr_sha":"1411888568eb8c72f1a939fe6076c3f61faaa630","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:57:16Z","started_at":"2026-09-18T15:57:16Z","completed_at":"2026-09-18T18:33:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.463439674Z","timing_metadata_checked_at":"2026-09-19T03:30:43.463439674Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.466385185Z","region_metadata_checked_at":"2026-09-19T03:30:43.466385185Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"fef1b259e6981712a9c624e9eab0c55d83f17aeb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T15:17:47Z","started_at":"2026-09-18T15:17:47Z","completed_at":"2026-09-18T18:36:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:35.860472443Z","timing_metadata_checked_at":"2026-09-19T03:30:35.860472443Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:35.864219257Z","region_metadata_checked_at":"2026-09-19T03:30:35.864219257Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1948be335692ad9a124313808e2157ccdb05236054f536818bd71b0afb7722f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"9e65f4af22bf7ec9953560fed8bb49590fa82fc63d7f9f6d24d707a1a503f81c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2657095b66e4cc164c41d453df5451025086cf7e54a24d404a46420fefe7d7e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"alert","signature_id":"035123031fa1d20ace889ddb3bd473206d7c64b2e0e69a0775e5e52e6c40e898","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:03:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"880ec577c2d949332cde1d90917ab779ddc5c423ce63817f14fb1fbbaae01d67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"alert","signature_id":"b2d9d84ce6e1c74ea14279f086b91ae4fc06d24e40b82a7ad2aa54e78ffa7d4b","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:04:42Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease governance-policy-framework in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease config-policy-controller in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"8c9023598adfb161da69a9a1b49b6b4e86180078bc9c1dcbe9aac1655663ab5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"a0f110873c9fc7db9f376a400bf0d8b83aa3eddcd644b62e99f4963c1f15aa39","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c46e909b0e6f4e8a1c9afd46392acc4d6cfdb0c1397ae5723a896270d73c4af7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"7e8e740680a98bd62be8f0f23d65fc40251d4d5f7742370deded696800b1b588","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5202a8e261350ac547743f88e12674d7803f9696bbba338e24743bc31c9862c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"52f64f30d917c25349ded7085a1164f0e2615c20f54359a71e907e0ab61c033a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100982457594548224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6858,"pr_state":"open","pr_sha":"150b9bf5d82a4bcc1f959e465ffe6769e739370b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:08:59Z","started_at":"2026-09-18T15:08:59Z","completed_at":"2026-09-18T18:15:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:35.595768767Z","timing_metadata_checked_at":"2026-09-19T03:30:35.595768767Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:35.599651282Z","region_metadata_checked_at":"2026-09-19T03:30:35.599651282Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100965684178587648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"dde72f49803d49335aafd35b01b79b6a92ac9d27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:08:34Z","started_at":"2026-09-18T15:08:34Z","completed_at":"2026-09-18T17:17:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:36.532213921Z","timing_metadata_checked_at":"2026-09-19T03:30:36.532213921Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:36.535544433Z","region_metadata_checked_at":"2026-09-19T03:30:36.535544433Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"88cc31ebdf51e981ac966aab5fe4333c0fee21a7","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T15:08:20Z","started_at":"2026-09-18T15:08:20Z","completed_at":"2026-09-18T15:49:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:50:44.615833908Z","timing_metadata_checked_at":"2026-09-18T15:50:44.615833908Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:50:44.618971719Z","region_metadata_checked_at":"2026-09-18T15:50:44.618971719Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"24e43b78ca91d5bc59c5953524fe878e8eb8d41d5a26884bb53457353386e07f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","occurred_at":"2026-09-18T15:08:20Z","failed_at":"provision","signature_id":"e4c3fb1587b09488a2265155d7a6787ca94e35b729bce621d58ebb8765ea7ef1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-18T15:40:00.669Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T15:42:51.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"name\\\": \\\"326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'e78fa208-8c43-45b6-a80c-57ad93c6a044'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4742016/providers/Microsoft.Resources/deployments/ka7tdccaiavlrtibmy3y5s7tfhcs6gpzxc6p4xe5pchjy-26091815bptz8\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: d9b78f81-a4e0-4b15-9e4d-75e6f5c342e3.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100972082392207360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T15:02:13Z","started_at":"2026-09-18T15:02:13Z","completed_at":"2026-09-18T17:27:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:37.673985331Z","timing_metadata_checked_at":"2026-09-19T03:30:37.673985331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:37.678980149Z","region_metadata_checked_at":"2026-09-19T03:30:37.678980149Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100972655304773632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T15:01:55Z","started_at":"2026-09-18T15:01:55Z","completed_at":"2026-09-18T17:36:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:37.325413146Z","timing_metadata_checked_at":"2026-09-19T03:30:37.325413146Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:37.32923696Z","region_metadata_checked_at":"2026-09-19T03:30:37.32923696Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T13:17:26Z","started_at":"2026-09-18T13:17:26Z","completed_at":"2026-09-18T15:21:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.925955013Z","timing_metadata_checked_at":"2026-09-18T15:41:21.925955013Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.929088124Z","region_metadata_checked_at":"2026-09-18T15:41:21.929088124Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"87ca00630dc338a1e8f9db92f36c7acce78945e58c03404050bd9733bacd7b92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","occurred_at":"2026-09-18T13:17:26Z","failed_at":"alert","signature_id":"d6f983561c2168d8dc1fb0bb24a4ce58dae5f6ea67b850648a086ef0cc8ced1e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T14:32:08Z\n Ended: 2026-09-18T14:39:08Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1716224-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1716224-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"91261d5b93a8d4039291003288ddae318a2a8506","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T12:56:34Z","started_at":"2026-09-18T12:56:34Z","completed_at":"2026-09-18T15:14:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.599107407Z","timing_metadata_checked_at":"2026-09-18T15:41:20.599107407Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.603177722Z","region_metadata_checked_at":"2026-09-18T15:41:20.603177722Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7112400e1b8f88fe771ccb3807b8cc948c099d4421e9d140be9b9294fad6bf07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","occurred_at":"2026-09-18T12:56:34Z","failed_at":"e2e","signature_id":"946ebd2cdc29e94011829e7ef21165a52d1801fd362344870c8d6a3e7c5269c4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100926281322336256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T12:33:53Z","started_at":"2026-09-18T12:33:53Z","completed_at":"2026-09-18T14:30:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.215063671Z","timing_metadata_checked_at":"2026-09-18T15:40:40.215063671Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.218157282Z","region_metadata_checked_at":"2026-09-18T15:40:40.218157282Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T11:57:07Z","started_at":"2026-09-18T11:57:07Z","completed_at":"2026-09-18T14:00:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.612275794Z","timing_metadata_checked_at":"2026-09-18T15:40:45.612275794Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.615271905Z","region_metadata_checked_at":"2026-09-18T15:40:45.615271905Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"99fbae33497987b2ca58c464dd69244aac980463b1109cfdccc4b526732c17d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","occurred_at":"2026-09-18T11:57:07Z","failed_at":"alert","signature_id":"85fef6c172f8238e32065ce9be94996092fd65499dbb48afe109749b4511663e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:52Z\n Ended: 2026-09-18T13:38:52Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6576768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6576768-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T11:54:51Z","started_at":"2026-09-18T11:54:51Z","completed_at":"2026-09-18T14:07:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.271599771Z","timing_metadata_checked_at":"2026-09-18T15:40:44.271599771Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.275509786Z","region_metadata_checked_at":"2026-09-18T15:40:44.275509786Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0b321804d1b8c11acab93194bb72fc88d0f4a78a49c438ee42b7c414ef5d9dde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"991b7eb9b2760771a64ba10515aeb5d1b17b39f0d0d776abbc32c3f1fb85abb3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (2 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:05:51Z\n Ended: 2026-09-18T13:34:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j\", pod=\"router-86c7fcbd7d-pjgqb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j/router-86c7fcbd7d-pjgqb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:07:43Z\n Ended: 2026-09-18T13:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster\", pod=\"router-7c5f44b8f5-2rxm6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster/router-7c5f44b8f5-2rxm6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"28bce12764c4b9c1d361a139c92d92bb565be67867f79583c26a96a6bda7f9aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"77db7532989dc335d44e3fd0e347138b7a4aa37045fd816ecfc3761a0789b616","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:44:43Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:45:42Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"39638348eb75451765766ab378ccb201ddf353daa0248edb4b62dfb6c29fb70d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"803dd3435e0189d03ce01d4d7de3f8e741a5ee356c8001e19b84595bbaa71298","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"409e38279d0775df031e734a590194632f5e380be587192046ffde98a8006e92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"4367b100b20534920c26dfaf7f8f61e0605f9aa71cbb971e026f2f07d8c0892b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"f71f9d0966fc26ad6ff3cd9a2f086d86818c4234b98d4c027fd22b942a6e7ba2","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:16:41Z\n Ended: 2026-09-18T13:44:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:17:41Z\n Ended: 2026-09-18T13:45:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"837b8358c268b512a4274ba3da67b785913fe6d7e617731945b4423b0c563171","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"b30257b5749c88e7acf4c108618d88d0f18563968e68f249086f8fef88556ee6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"88d0baaeb3ce553af8f103da0477073c87b8bd2e014b51494b3e15118a9a1525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"65b5dbb9967304eedefd97b212163b3f8af9b0b2bb0188919a3d12f3a6538fc2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de254d309520eb7dcc15cb4c173bd21c1b68dfea356b64131ea7e4a21eaaa4c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"524cdf887bee162758b9f6eca5510d2578bb6ecf654e9948629c9452eaf1782b","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100913554960945152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"open","pr_sha":"acfa8be9cc45d0569ea8da4e90158944e7541735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T11:43:14Z","started_at":"2026-09-18T11:43:14Z","completed_at":"2026-09-18T13:42:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.113325155Z","timing_metadata_checked_at":"2026-09-18T15:41:07.113325155Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.11742317Z","region_metadata_checked_at":"2026-09-18T15:41:07.11742317Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100898990957531136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6694,"pr_state":"closed","pr_sha":"f1437e7522fb6efcd77f296c45896ba121660db7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T10:45:26Z","started_at":"2026-09-18T10:45:26Z","completed_at":"2026-09-18T10:57:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:18.538824492Z","timing_metadata_checked_at":"2026-09-18T15:40:18.538824492Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:18.542726706Z","region_metadata_checked_at":"2026-09-19T03:30:33.749012857Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"159e2b1150b1ec7c3f974e57cd7712f7f278dc6df04ca2922e456ef7b1b80ed7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100898990957531136","occurred_at":"2026-09-18T10:45:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100892388036710400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6879,"pr_state":"closed","pr_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","final_merged_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T10:19:12Z","started_at":"2026-09-18T10:19:12Z","completed_at":"2026-09-18T12:27:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.766218983Z","timing_metadata_checked_at":"2026-09-18T15:40:29.766218983Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.769670695Z","region_metadata_checked_at":"2026-09-18T15:40:29.769670695Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"91261d5b93a8d4039291003288ddae318a2a8506","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:00:26Z","started_at":"2026-09-18T10:00:26Z","completed_at":"2026-09-18T12:04:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.436005317Z","timing_metadata_checked_at":"2026-09-18T15:41:20.436005317Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.439499329Z","region_metadata_checked_at":"2026-09-18T15:41:20.439499329Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"43a0402625e5406ea76d4f94e1c32004477778ea8ad7365eb990e95245f7b6c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","occurred_at":"2026-09-18T10:00:26Z","failed_at":"alert","signature_id":"c3bce50243ff744e90a9975f976bf7851614b19682c44e007ffa00d7c6f101f9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:02:41Z\n Ended: 2026-09-18T11:23:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5658496-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stcj8snr7hlhv8hfthg1emt8dhhfded-j1a6w4y5x8t0x1d\", pod=\"router-769f87966d-khp89\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stcj8snr7hlhv8hfthg1emt8dhhfded-j1a6w4y5x8t0x1d/router-769f87966d-khp89 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T11:04:44Z\n Ended: 2026-09-18T11:23:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5658496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stckihvk61lv7eojs5hbnlalilj7v1i-agg-cluster\", pod=\"router-7c9ffb6845-wczwq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stckihvk61lv7eojs5hbnlalilj7v1i-agg-cluster/router-7c9ffb6845-wczwq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T09:55:36Z","started_at":"2026-09-18T09:55:36Z","completed_at":"2026-09-18T12:07:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.702185479Z","timing_metadata_checked_at":"2026-09-18T15:41:15.702185479Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.706062693Z","region_metadata_checked_at":"2026-09-18T15:41:15.706062693Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"778d8ce6a0c00a1e3bda2fb387049c59cc3d888b0904fe2e8c3e42f76d242cb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","occurred_at":"2026-09-18T09:55:36Z","failed_at":"alert","signature_id":"fcdb5ce6d5eb5426a483be7faf7e19f591fbf253b3ebf7b8ca13d6e2fd0d11f4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:09:41Z\n Ended: 2026-09-18T11:20:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9620352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stclp139sl99taqsqoij7cie2hd4cpk-v20260901\", pod=\"router-5b84bc6565-scjwf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stclp139sl99taqsqoij7cie2hd4cpk-v20260901/router-5b84bc6565-scjwf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T09:52:14Z","started_at":"2026-09-18T09:52:14Z","completed_at":"2026-09-18T12:14:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.781146879Z","timing_metadata_checked_at":"2026-09-18T15:40:08.781146879Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:08.784934493Z","region_metadata_checked_at":"2026-09-18T15:40:08.784934493Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"6926365da1644e2f91dfd7c5fbe3a79ab84cf7c2c913a442441f97c35df1ad0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","failed_at":"alert","signature_id":"4e84895295e21c9ae798ddae187f381094acb88d5bb19172060995a42efdfbcd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:21:18Z\n Ended: 2026-09-18T11:37:18Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3790208-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3790208-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"da069b10122f51fa80e0ae2bcf494afdef048174e2a89a3b93080f1a41097cda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","failed_at":"alert","signature_id":"002a8aff34d1219bae02653dd6f6335a032c8c53538dc0e88e47e06a4653de19","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:14:28Z\n Ended: 2026-09-18T11:23:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3790208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stcn6svivskocqagj4qaaqujiq0cekp-m1e6y0x7h1c1n8p\", pod=\"router-748fddb496-fcbbx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stcn6svivskocqagj4qaaqujiq0cekp-m1e6y0x7h1c1n8p/router-748fddb496-fcbbx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T09:21:30Z","started_at":"2026-09-18T09:21:30Z","completed_at":"2026-09-18T11:21:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.520475064Z","timing_metadata_checked_at":"2026-09-18T15:40:45.520475064Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.524059977Z","region_metadata_checked_at":"2026-09-18T15:40:45.524059977Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"aa9e1643ab2e037088a071c942816832f5dd1d5c40161259238d64e030d67554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"alert","signature_id":"7eefe8fce4efeddab6a7bddfd5f7e5cb50289671f56d6e25708c25b9305ce796","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T10:18:52Z\n Ended: 2026-09-18T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8771328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbuceofnif99sq6etloedbqkpc5qh9-u1a3e7t8x9z9b1l\", pod=\"router-b6965d7dc-jzrb7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbuceofnif99sq6etloedbqkpc5qh9-u1a3e7t8x9z9b1l/router-b6965d7dc-jzrb7 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"d5dea3392eb2163b7d8f1d40e9615b3d55175ba5913ecb983521f10c10fa03a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"alert","signature_id":"077ee51cbdfb7388e2c8c28578e14f7c26aad49cf417b4ab9a95eda6e094ce1d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T10:27:52Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T10:28:56Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"fdd4ba74c769ebfd11719783a6a41b89c38eea91e67ee842d218c8207011b514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"e2e","signature_id":"f590a6d9decf3d5d50d5dfcef242c5c8af6ebdbb758d3e90d6fae5136781bae3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:44:01Z","started_at":"2026-09-18T08:44:01Z","completed_at":"2026-09-18T10:35:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.135552482Z","timing_metadata_checked_at":"2026-09-18T15:40:44.135552482Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.138715793Z","region_metadata_checked_at":"2026-09-18T15:40:44.138715793Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"955bb3878fbb804e63ad31f56c01fd2e666068f98c58de7ff1ea549e8b5da0aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"alert","signature_id":"be273c1646946db878b565bca63154b691913899e238a33aee949d4d46ae8e4f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:36:00Z\n Ended: 2026-09-18T10:18:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3745536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stba81rimklhsg3cr5ktr5tf749cko1-u3d9c8l0y5d3t2z\", pod=\"router-5b649bbbf7-4cvr4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stba81rimklhsg3cr5ktr5tf749cko1-u3d9c8l0y5d3t2z/router-5b649bbbf7-4cvr4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"985a99cacb656894b6047e4124ce83ff8d4d737c41dac3e33a006cde078ecdf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"78800d7f9bd7ce60d82097401bbc1a1b346c5eb770d7eeedc635c6d48abac870","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-l7rx685m27vp/cluster-candidate-4-20-4l4f7q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ac60\u003e: \n failed to create HCP cluster cluster-candidate-4-20-4l4f7q: failed waiting for cluster=\"cluster-candidate-4-20-4l4f7q\" in resourcegroup=\"rg-candidate-4-20-l7rx685m27vp\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"name\": \"d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:24:34.52165977Z\",\n \"endTime\": \"2026-09-18T09:53:41.437046134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 21m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-l7rx685m27vp/cluster-candidate-4-20-4l4f7q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ac60\u003e: \n failed to create HCP cluster cluster-candidate-4-20-4l4f7q: failed waiting for cluster=\"cluster-candidate-4-20-4l4f7q\" in resourcegroup=\"rg-candidate-4-20-l7rx685m27vp\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"name\": \"d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:24:34.52165977Z\",\n \"endTime\": \"2026-09-18T09:53:41.437046134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 21m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c570e2e00a3bb9b3ebf76b01680e0fe0c751b734150705cdfbf1f0aa48f7c616","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"ba73d1a54ec8e3a0b02523dbd4276ff942ead7e6f9b4e52703889f225a290272","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000faea40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdl2drfvfhtk\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\",\n \"name\": \"4fc5c348-7117-4850-997b-20eb599df30e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:25:07.592750123Z\",\n \"endTime\": \"2026-09-18T09:54:11.523791716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000faea40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdl2drfvfhtk\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\",\n \"name\": \"4fc5c348-7117-4850-997b-20eb599df30e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:25:07.592750123Z\",\n \"endTime\": \"2026-09-18T09:54:11.523791716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0535b166f31d721ee73bca028f5c2a263e713a6075f4c754234b073b9f48232","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"open","pr_sha":"acfa8be9cc45d0569ea8da4e90158944e7541735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:38:28Z","started_at":"2026-09-18T08:38:28Z","completed_at":"2026-09-18T10:37:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.016824507Z","timing_metadata_checked_at":"2026-09-18T15:41:07.016824507Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.022435828Z","region_metadata_checked_at":"2026-09-18T15:41:07.022435828Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2423d0d06a02e9c0ba4df730fd696a975e13e7e397a1f473b152ad31b5e16db0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"alert","signature_id":"d6db3f97bdaa9f3b8874f62b7846be7bdd694b5e229a6dfe93f7bd212a0dcb02","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:41:54Z\n Ended: 2026-09-18T10:11:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5368576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbcs84sip93t6dffm9229aerut2acv-z8r9c8z3p9s9m1i\", pod=\"router-d76b55d7-rgg2v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbcs84sip93t6dffm9229aerut2acv-z8r9c8z3p9s9m1i/router-d76b55d7-rgg2v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T09:42:48Z\n Ended: 2026-09-18T10:11:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5368576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbd564riaqku802qrnf0q9bc8r9vik-private-kas\", pod=\"router-85dc58d9b7-tsdrp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbd564riaqku802qrnf0q9bc8r9vik-private-kas/router-85dc58d9b7-tsdrp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cb9dd419229483ed402cd42b90130ff4f3bbfbe90e59cf993cdccf94f29ee124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"e2e","signature_id":"bb9a6198465bbe86bad7f0728097a5e8139953db856c6c8aad613b2d535d245f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcaa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5jssdfr4s6w\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"name\": \"92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:20.728673075Z\",\n \"endTime\": \"2026-09-18T09:58:22.849626679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcaa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5jssdfr4s6w\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"name\": \"92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:20.728673075Z\",\n \"endTime\": \"2026-09-18T09:58:22.849626679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d91a2664b239a950a30ac6a20b3487dd4fbe89eeb800e668ff0ded3b40618f20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"e2e","signature_id":"2e53b9a5382ed0f0493cfc0c48e9bf7aa226835370880aa8b0e3c984f3d43f8d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8kv74t8cs9l5\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"name\": \"4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:55.514391889Z\",\n \"endTime\": \"2026-09-18T09:59:02.761026905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8kv74t8cs9l5\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"name\": \"4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:55.514391889Z\",\n \"endTime\": \"2026-09-18T09:59:02.761026905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863236734717952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T08:23:22Z","started_at":"2026-09-18T08:23:22Z","completed_at":"2026-09-18T10:34:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.989207567Z","timing_metadata_checked_at":"2026-09-18T15:40:34.989207567Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.992552479Z","region_metadata_checked_at":"2026-09-18T15:40:34.992552479Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"8c23f9b0c37e8f10e070fd2c1d0831ee8fd99e56","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:23:07Z","started_at":"2026-09-18T08:23:07Z","completed_at":"2026-09-18T10:24:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:39.485293545Z","timing_metadata_checked_at":"2026-09-18T15:40:39.485293545Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:39.489133258Z","region_metadata_checked_at":"2026-09-18T15:40:39.489133258Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2028dd15cd9a1813f35b867a2d1aed5557176f389b8b74009f14759aed2232e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"1ff84f075d12052118fcca1c651ed549b6c5f0208069e0615f46732cae26b19e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp"}},{"row_id":"2be3e643fa3f57bb4de7eb40d271fd16091009ad89f2a29e90e00b3230c7852c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"7193ae4d62017e53f8c9422389a5405738a2e8a036e8555e76291f330d89920e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"8e6e197a9c1d0541b3d16ee31945ea91c35418b1bc264b205fca4968923fafec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:27:59Z\n Ended: 2026-09-18T09:34:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6818048-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9c239768e853cd4f0bde3ead855329c5aa91e04279ccb76ac5426e3a965ee90f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"alert","signature_id":"f47f0b6edf615e207ea20e3174c9c4b6c3860a8cd322039375d46696bfbe5705","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:24:01Z\n Ended: 2026-09-18T09:49:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6818048-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stb4ma2bs7029nmfb05hiqpcdsgabul-w1y0u6t1v1n3z9l\", pod=\"router-6bd77b545b-pvt7r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stb4ma2bs7029nmfb05hiqpcdsgabul-w1y0u6t1v1n3z9l/router-6bd77b545b-pvt7r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e43e42f4a6d0e49318031ab937af0f655181064f910d931a061226af618cc553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"001338be47cf8dcaa3fc3c500d6a6dec5a631054e1ca9f9d544c49d235b2dcb7","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9a600\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pwncww98jzm8\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"name\": \"145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:11:52.011321272Z\",\n \"endTime\": \"2026-09-18T09:40:57.964346748Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9a600\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pwncww98jzm8\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"name\": \"145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:11:52.011321272Z\",\n \"endTime\": \"2026-09-18T09:40:57.964346748Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863036339261440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:22:32Z","started_at":"2026-09-18T08:22:32Z","completed_at":"2026-09-18T08:23:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.001130298Z","timing_metadata_checked_at":"2026-09-18T15:40:44.001130298Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:44.004775711Z","region_metadata_checked_at":"2026-09-19T03:30:37.157285126Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2e6f85effdaeb3112437499df2f3a95965bc9163b1e03939a870a0232f47889c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863036339261440","occurred_at":"2026-09-18T08:22:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:21:29Z","started_at":"2026-09-18T08:21:29Z","completed_at":"2026-09-18T10:18:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.318614845Z","timing_metadata_checked_at":"2026-09-18T15:41:34.318614845Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.321862257Z","region_metadata_checked_at":"2026-09-18T15:41:34.321862257Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6c210d1a83ddb761e4bfb2fba935a2a001fb9c36576fa71840fc19636c6d912b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","occurred_at":"2026-09-18T08:21:29Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:23:28Z\n Ended: 2026-09-18T09:39:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8030592-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7003,"pr_state":"open","pr_sha":"5ef6466e2a8ec2041b8b57cd5821ccdc91bbdd88","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:07:05Z","started_at":"2026-09-18T08:07:05Z","completed_at":"2026-09-18T09:54:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:22.074694851Z","timing_metadata_checked_at":"2026-09-18T15:41:22.074694851Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:22.077840163Z","region_metadata_checked_at":"2026-09-18T15:41:22.077840163Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f2b83989eb95efb2403df14346d44f01552bcb574650cc9c2db78d28140feaec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","occurred_at":"2026-09-18T08:07:05Z","failed_at":"e2e","signature_id":"4553dd78f39dc989ce886fb5e82604c7aa8be953927622430a5564e21a9cc7f0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100856808020119552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6962,"pr_state":"closed","pr_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","final_merged_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T07:57:47Z","started_at":"2026-09-18T07:57:47Z","completed_at":"2026-09-18T09:57:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.306908853Z","timing_metadata_checked_at":"2026-09-18T15:40:58.306908853Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.310468766Z","region_metadata_checked_at":"2026-09-18T15:40:58.310468766Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T06:57:25Z","started_at":"2026-09-18T06:57:25Z","completed_at":"2026-09-18T09:16:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.993207072Z","timing_metadata_checked_at":"2026-09-18T15:40:40.993207072Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.996500084Z","region_metadata_checked_at":"2026-09-18T15:40:40.996500084Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ee4aebdad12a21a86688d0b3660ead46e494170673ae2c2e6e13761cebb14f76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","occurred_at":"2026-09-18T06:57:25Z","failed_at":"alert","signature_id":"0d95d8184561f593338e86b1c11b5ad3f6442e5263869a130eac56ca5e35b079","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:31:06Z\n Ended: 2026-09-18T08:38:05Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5294976-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5294976-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T06:57:14Z","started_at":"2026-09-18T06:57:14Z","completed_at":"2026-09-18T09:19:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.949440129Z","timing_metadata_checked_at":"2026-09-18T15:41:12.949440129Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.952906042Z","region_metadata_checked_at":"2026-09-18T15:41:12.952906042Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"5d8b5e5c733e92bb36be533024a94ed93a2579559c695da763da987f7d7bdb48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","failed_at":"alert","signature_id":"b8ef7bc40a5e42cb1d9a25c2c491ed35aa8060746d13168cf09c7ed965a89066","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:33:35Z\n Ended: 2026-09-18T08:39:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4018048-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4018048-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fdfc1f8ee548108afcd7a729dce171c7ad549661d33c82f685b2b89ceeabf469","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","failed_at":"alert","signature_id":"7b986d6b9e982cdf65d80f902e9563f9d1125a3f54f3c1a12116db416ae5f10d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:44:25Z\n Ended: 2026-09-18T08:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4018048-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sta451luu6i39gfi3s8vomm21o2b1lj-v1u6q6s5g4j0h3p\", pod=\"kube-apiserver-5959d9c69f-k9pqp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sta451luu6i39gfi3s8vomm21o2b1lj-v1u6q6s5g4j0h3p/kube-apiserver-5959d9c69f-k9pqp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100833415631736832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"a20dd280dd6b7499cf853051219487314ef1e460","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:24:52Z","started_at":"2026-09-18T06:24:52Z","completed_at":"2026-09-18T08:23:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.356877104Z","timing_metadata_checked_at":"2026-09-18T15:40:31.356877104Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:31.360236816Z","region_metadata_checked_at":"2026-09-18T15:40:31.360236816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c0ebd76b50205ad901e577ade460296f93b4079e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T06:09:33Z","started_at":"2026-09-18T06:09:33Z","completed_at":"2026-09-18T08:14:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.906877307Z","timing_metadata_checked_at":"2026-09-18T15:41:01.906877307Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.909999818Z","region_metadata_checked_at":"2026-09-18T15:41:01.909999818Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"058d4d1662f3ff9282ca6e8faa2c2cae8d2bf7201d772467499d9a0089725839","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"79534399cf979be34c0ffb48ddc762d6841c4a16f58dc222924b468f595a7adb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ff8902bcd442597127933fe639d5a91a9e1d31e98c38e00f7d9192969447637","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"abcb5fca9b88f5de84bce4c63659906c7070748eea55c112da30810936b0fc97","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-895tpwlfvzpg/cluster-candidate-5-0-jpmgmm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e760\u003e: \n failed to create HCP cluster cluster-candidate-5-0-jpmgmm: failed waiting for cluster=\"cluster-candidate-5-0-jpmgmm\" in resourcegroup=\"rg-candidate-5-0-895tpwlfvzpg\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"name\": \"d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:57:19.894043957Z\",\n \"endTime\": \"2026-09-18T07:26:23.420290614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-895tpwlfvzpg/cluster-candidate-5-0-jpmgmm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e760\u003e: \n failed to create HCP cluster cluster-candidate-5-0-jpmgmm: failed waiting for cluster=\"cluster-candidate-5-0-jpmgmm\" in resourcegroup=\"rg-candidate-5-0-895tpwlfvzpg\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"name\": \"d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:57:19.894043957Z\",\n \"endTime\": \"2026-09-18T07:26:23.420290614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24e519c9ea99624bed46d72420daea446f43d0f46833aed446771e34255ebefd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"cc794ab286f7ef163ad70de687149ee27cd59dcc48d24d280de823796404177d","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"45eb23e248127a5fd1cf73f5a23034867813c0e52e1e7dd6e30bab10faa48bb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"782ec114fda729c308336015b9ba3ea7dba7299080a2ca46beda618ff211a392","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080af80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4pnfzqpsrckd\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"name\": \"540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:58:33.125357484Z\",\n \"endTime\": \"2026-09-18T07:27:33.397040852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080af80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4pnfzqpsrckd\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"name\": \"540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:58:33.125357484Z\",\n \"endTime\": \"2026-09-18T07:27:33.397040852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"af42e922170b2d99c9383756f170a9227b8fb5aaf3e36571cffd112ac2677d29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"8db752092d73d2ebfaaaef7b1150f759438bf639db4c2c7645ebdce228946dcf","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100828394844327936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:04:55Z","started_at":"2026-09-18T06:04:55Z","completed_at":"2026-09-18T08:14:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.536925977Z","timing_metadata_checked_at":"2026-09-18T15:40:59.536925977Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.539776288Z","region_metadata_checked_at":"2026-09-18T15:40:59.539776288Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827927296872448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"96f38fd5a277adea0aeec323ef5685124af19024","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:03:03Z","started_at":"2026-09-18T06:03:03Z","completed_at":"2026-09-18T08:09:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.149508965Z","timing_metadata_checked_at":"2026-09-18T15:40:29.149508965Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.153306579Z","region_metadata_checked_at":"2026-09-18T15:40:29.153306579Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T06:01:39Z","started_at":"2026-09-18T06:01:39Z","completed_at":"2026-09-18T07:55:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.162062581Z","timing_metadata_checked_at":"2026-09-18T15:41:34.162062581Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.165221793Z","region_metadata_checked_at":"2026-09-18T15:41:34.165221793Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"19cc5011eb4bf8fc2743655dddceed3860753b855db9569c6b609dac39300849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"e2e","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"510d2ef135f1ec6cd6b28e6a8dff9c37612315d8ac546638abd6539fccc9f0e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"e2e","signature_id":"25746549fcd7e3d493a2e9010aa4c2f674167bdbce01dd388ff30ebecae482d7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"96d68ed0e6539b9919596989a03ac86d6279a7ffed0bce91ea23ad81d6388996","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"alert","signature_id":"56136cc3eeacd2663934d686bb2d9b29fb8b76aaa1ea7443d05087ccb6f12b10","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:59:52Z\n Ended: 2026-09-18T07:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91kv9p4h3i5f2qjatf1ddh7hg451l-u6e3o9h6n4t0q1m\", pod=\"router-56dbc9cf68-flvzm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91kv9p4h3i5f2qjatf1ddh7hg451l-u6e3o9h6n4t0q1m/router-56dbc9cf68-flvzm has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T07:00:48Z\n Ended: 2026-09-18T07:18:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91ihbjq7vgovat1v96s2dgh6gc8k8-ingress-cert-le\", pod=\"router-866f65df77-vw5dl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91ihbjq7vgovat1v96s2dgh6gc8k8-ingress-cert-le/router-866f65df77-vw5dl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-18T07:00:48Z\n Ended: 2026-09-18T07:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91vs1vhcp81dngelnmuif63cd1p56-private-kas\", pod=\"router-6449dfbd8c-5w442\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91vs1vhcp81dngelnmuif63cd1p56-private-kas/router-6449dfbd8c-5w442 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100825645108957184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:53:59Z","started_at":"2026-09-18T05:53:59Z","completed_at":"2026-09-18T06:03:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.572008709Z","timing_metadata_checked_at":"2026-09-18T15:41:15.572008709Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:15.57529412Z","region_metadata_checked_at":"2026-09-19T03:30:40.600884587Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"22e97c2e288eaaab62face32d60d489ff031945e6aa062cea0ce6394812e8872","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100825645108957184","occurred_at":"2026-09-18T05:53:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6962,"pr_state":"closed","pr_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","final_merged_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:20:59Z","started_at":"2026-09-18T05:20:59Z","completed_at":"2026-09-18T07:10:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.18925523Z","timing_metadata_checked_at":"2026-09-18T15:40:58.18925523Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.192641342Z","region_metadata_checked_at":"2026-09-18T15:40:58.192641342Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"48fc808570ba8478c7da5f5a570807c3fc61e66dfbdc6c290988b25eed84bb9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","occurred_at":"2026-09-18T05:20:59Z","failed_at":"e2e","signature_id":"fd810f6089fd17b9ae89f5ca382f4365705ce74f6259d695bca5e674d7783e53","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817174481342464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:20:20Z","started_at":"2026-09-18T05:20:20Z","completed_at":"2026-09-18T06:09:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.802472898Z","timing_metadata_checked_at":"2026-09-18T15:41:12.802472898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.812156633Z","region_metadata_checked_at":"2026-09-18T15:41:12.812156633Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ccbfb930772320d5e8f6f15fc7bd93741f16a645865155f636c41269986ea602","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817174481342464","occurred_at":"2026-09-18T05:20:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6813,"pr_state":"open","pr_sha":"4a5aea6cd3a0485cade670b99e2a3f7a2e59ebbb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:13:05Z","started_at":"2026-09-18T05:13:05Z","completed_at":"2026-09-18T07:09:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.033420457Z","timing_metadata_checked_at":"2026-09-18T15:40:25.033420457Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.036653268Z","region_metadata_checked_at":"2026-09-18T15:40:25.036653268Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"74ed0f71392d5582daeceec0dc12cc85e38ab0e9c2693ba7191fa2ed8345e536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","failed_at":"e2e","signature_id":"1b937eb2ed1eae4fc31ae0598f8de6ce2076a500845fc3ea7dcd762870526b2e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0009cb668\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6k7xjqv9k7sk\" to finish creating: GET https://rp.j1065216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"name\": \"6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:12:51.736703148Z\",\n \"endTime\": \"2026-09-18T06:31:55.141914948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0009cb668\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6k7xjqv9k7sk\" to finish creating: GET https://rp.j1065216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"name\": \"6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:12:51.736703148Z\",\n \"endTime\": \"2026-09-18T06:31:55.141914948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e3cf9384daecc3451f8c4882fa99d01a23436abc728c976e75e44526ae05e9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","failed_at":"e2e","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6157,"pr_state":"open","pr_sha":"d4af5249d6e0478ea45dc795343aaa530c0f147b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:12:47Z","started_at":"2026-09-18T05:12:47Z","completed_at":"2026-09-18T07:19:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.885570852Z","timing_metadata_checked_at":"2026-09-18T15:40:04.885570852Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.888765463Z","region_metadata_checked_at":"2026-09-18T15:40:04.888765463Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1d2ec301c869b7058e1550ed7eef64aa4c80e53281a61a75e279669c24f17ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"4fb56cfa0c5d81c59f21dba90bd2b4aeaa2b18f5202205163a7a19b83e403f4e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2238aef8fd05ba5d9f163445be374775a863bb390c7296dbc2733ec0a60e5e06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"alert","signature_id":"49184b986d87df135df2fc3dcc64465876a9918af7c03bd147e1ea6126eccebf","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:22:09Z\n Ended: 2026-09-18T06:28:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0748928-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0748928-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"31bdad879e96505d0a8491d793dba34142cef8ede9e4f2fcd6b3ae74df94b79f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"alert","signature_id":"476aed8123ee528db23f3663c1332194314ed537bec78310ea649964cd6c166f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T07:08:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0748928-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9hl75\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9hl75 (namespace arobit) on cluster ci01-j0748928-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"7a969b98fae0731b36cb6bf2edab5476bfc770f0148e570c222bfc1fc55de8d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"0d8835203476a034717107aadec991b28272c94910dad48bde3bb462bb56ac9b","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bf9085c5865b518467942404d4892e807542a93a01922bb60f84e377306713e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"06ba3d2a9582d2a18869bdd5ae1df4321a4d0977a7be4bdd0e3cc1af2ec0608a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","failure_pattern":"\"error\": {","search_query":"\"error\": {"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815135416258560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"728300dde5778eae2cc566e635f9026903fc274b","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:12:14Z","started_at":"2026-09-18T05:12:14Z","completed_at":"2026-09-18T05:35:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.429124592Z","timing_metadata_checked_at":"2026-09-18T15:41:15.429124592Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.440679834Z","region_metadata_checked_at":"2026-09-18T15:41:15.440679834Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c1000a36e27a701f2160dcd785daa85c641b7ac04049d77fb5874c75fb871f9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815135416258560","occurred_at":"2026-09-18T05:12:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815091837440000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T05:12:03Z","started_at":"2026-09-18T05:12:03Z","completed_at":"2026-09-18T07:06:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.171413155Z","timing_metadata_checked_at":"2026-09-18T15:40:24.171413155Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:24.177127876Z","region_metadata_checked_at":"2026-09-18T15:40:24.177127876Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:01:15Z","started_at":"2026-09-18T05:01:15Z","completed_at":"2026-09-18T07:06:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.638509365Z","timing_metadata_checked_at":"2026-09-18T15:40:08.638509365Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:08.642040978Z","region_metadata_checked_at":"2026-09-18T15:40:08.642040978Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"fac7c2814ab8e43eaa9ded8dba266bbff2ce61f4f6d45594d0fd1ccf378fdfd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","occurred_at":"2026-09-18T05:01:15Z","failed_at":"alert","signature_id":"e85df63033962c2849bfc318fc85b0a8cba9948dc2466a5fa45b6b2724994bfb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:00:50Z\n Ended: 2026-09-18T06:20:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2458624-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st858a4pvmlmac4uvehhfkife65pian-u9l5q3l8s3c3d7s\", pod=\"router-d9f7df5f5-npvv8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st858a4pvmlmac4uvehhfkife65pian-u9l5q3l8s3c3d7s/router-d9f7df5f5-npvv8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100811166501572608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T04:56:27Z","started_at":"2026-09-18T04:56:27Z","completed_at":"2026-09-18T07:01:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:47.107173571Z","timing_metadata_checked_at":"2026-09-18T15:40:47.107173571Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:47.110384283Z","region_metadata_checked_at":"2026-09-18T15:40:47.110384283Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100794344981139456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6930,"pr_state":"closed","pr_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","final_merged_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T03:49:37Z","started_at":"2026-09-18T03:49:37Z","completed_at":"2026-09-18T05:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.041950448Z","timing_metadata_checked_at":"2026-09-18T15:41:34.041950448Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.045422161Z","region_metadata_checked_at":"2026-09-18T15:41:34.045422161Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T03:21:45Z","started_at":"2026-09-18T03:21:45Z","completed_at":"2026-09-18T05:18:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.085262804Z","timing_metadata_checked_at":"2026-09-18T15:40:40.085262804Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.088495616Z","region_metadata_checked_at":"2026-09-18T15:40:40.088495616Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"3e65bdb606fff0e5981bdba342a4791f3d2fa1e38c8f50c107ed42a823bbe88c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"e2e","signature_id":"89409b5febc6093a9cefc9b6ca32afd019b45b211957e3bd23fe77c3eda373f3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93fc43fa475e4e277bb2bc7fad23f329306323c4ccc6329418ef90279c7fad2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"b14a5290094a9e0d25292157800f9158e06c44b980a0e58131fba650ea86837b","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:31:05Z\n Ended: 2026-09-18T04:48:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e333cab1-7b1b-43d4-a2a8-c38ce512c9fe\"\n Description: Pod ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"a32bb23b9959f8e7fb533a763afb0ee6f3f0488034bca1a4c24857e25b84f5e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:18:48Z\n Ended: 2026-09-18T04:37:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2362240-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"aa045988aa28993d83b618eef4ca84ea8647d0e81961c1247861c45b4948742d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"32caccfe24784ea73efa24f662aed9cbdd2ff753b7f3457b2fae68c38d3d3c3d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:29:08Z\n Ended: 2026-09-18T04:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2r9f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T04:29:08Z\n Ended: 2026-09-18T04:55:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2r9f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6984,"pr_state":"closed","pr_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","final_merged_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:11:35Z","started_at":"2026-09-18T03:11:35Z","completed_at":"2026-09-18T03:46:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.171007071Z","timing_metadata_checked_at":"2026-09-18T15:41:09.171007071Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.174145883Z","region_metadata_checked_at":"2026-09-18T15:41:09.174145883Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bfe862bede5f150ebf4ee6361d1b091eea5de565399b30f38ee0007fd9f2c344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","occurred_at":"2026-09-18T03:11:35Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-18T03:37:37.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T03:37:38.393Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-18T03:37:39.053Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1401344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298\ntime=2026-09-18T03:40:39.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298/operationStatuses/08584119042267536349\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-sr7huchijtjyg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1401344-mgmt-1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6984,"pr_state":"closed","pr_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","final_merged_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:08:52Z","started_at":"2026-09-18T03:08:52Z","completed_at":"2026-09-18T05:05:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.915554293Z","timing_metadata_checked_at":"2026-09-18T15:41:33.915554293Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.918892505Z","region_metadata_checked_at":"2026-09-18T15:41:33.918892505Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"3d70f2e86360715a46ec9ffc99c890cec9d30c215df5fed8e628acfbceddd40b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"alert","signature_id":"4750aa456ebf384aa187370c588c7897567d97246241f2ecee3733d106d2cd85","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:02:41Z\n Ended: 2026-09-18T04:32:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6907648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6e0gt56ef7jnfg7trhv4ir2638kjr-n2s7g5a5a5z9k1l\", pod=\"router-7d99594d96-6l44r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6e0gt56ef7jnfg7trhv4ir2638kjr-n2s7g5a5a5z9k1l/router-7d99594d96-6l44r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"788ccc521a93ece0184485b0fec366b00c6d4b00fc52518c2edc249a9588e08a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"a2d0482738306402a9aee374779b77d199fad57d181dc39a001fc3fa8cf50f3a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-8rxkwjmg9fdn/cluster-candidate-4-22-8z2p5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017682a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8z2p5b: failed waiting for cluster=\"cluster-candidate-4-22-8z2p5b\" in resourcegroup=\"rg-candidate-4-22-8rxkwjmg9fdn\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"name\": \"a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.402664583Z\",\n \"endTime\": \"2026-09-18T04:19:53.994096677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-8rxkwjmg9fdn/cluster-candidate-4-22-8z2p5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017682a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8z2p5b: failed waiting for cluster=\"cluster-candidate-4-22-8z2p5b\" in resourcegroup=\"rg-candidate-4-22-8rxkwjmg9fdn\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"name\": \"a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.402664583Z\",\n \"endTime\": \"2026-09-18T04:19:53.994096677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5551cdba1db795af316c870fe01077a6171db4f39637bb55efbc729a8f979fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"82775d4e1d46d1566b3f4eab5619ed61b625107edb899f3a6d4e9d4864e726a1","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--lv2mjp29lzrs/cluster-hs-pre-xxp2zq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aef7c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xxp2zq: failed waiting for cluster=\"cluster-hs-pre-xxp2zq\" in resourcegroup=\"rg-hs-pre--lv2mjp29lzrs\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\",\n \"name\": \"da216ffc-9136-436b-b01d-92158b449e7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:26.310458979Z\",\n \"endTime\": \"2026-09-18T04:19:33.969303507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--lv2mjp29lzrs/cluster-hs-pre-xxp2zq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aef7c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xxp2zq: failed waiting for cluster=\"cluster-hs-pre-xxp2zq\" in resourcegroup=\"rg-hs-pre--lv2mjp29lzrs\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\",\n \"name\": \"da216ffc-9136-436b-b01d-92158b449e7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:26.310458979Z\",\n \"endTime\": \"2026-09-18T04:19:33.969303507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb3120bfdce23eefa3b09de56c92ea4981f8b18405e705990a83a0a96ae4d4b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"925fc6ad3116b3a4af4d42c985323051e3b386c99d2bbff7d1917ed9c6b7ef67","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-wj65mtt7sqkj/cluster-candidate-4-21-pk5bmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181e300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pk5bmj: failed waiting for cluster=\"cluster-candidate-4-21-pk5bmj\" in resourcegroup=\"rg-candidate-4-21-wj65mtt7sqkj\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"name\": \"62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.884252443Z\",\n \"endTime\": \"2026-09-18T04:19:53.826763956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-wj65mtt7sqkj/cluster-candidate-4-21-pk5bmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181e300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pk5bmj: failed waiting for cluster=\"cluster-candidate-4-21-pk5bmj\" in resourcegroup=\"rg-candidate-4-21-wj65mtt7sqkj\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"name\": \"62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.884252443Z\",\n \"endTime\": \"2026-09-18T04:19:53.826763956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee75403d59abdaf958d8493407c062f18590022f6d80b33ca30dc123368aea31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:12:00Z\n Ended: 2026-09-18T04:22:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6907648-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100783081760755712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T03:04:51Z","started_at":"2026-09-18T03:04:51Z","completed_at":"2026-09-18T05:11:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.457172589Z","timing_metadata_checked_at":"2026-09-18T15:41:03.457172589Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:03.460414901Z","region_metadata_checked_at":"2026-09-18T15:41:03.460414901Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:01:52Z","started_at":"2026-09-18T03:01:52Z","completed_at":"2026-09-18T05:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.778959001Z","timing_metadata_checked_at":"2026-09-18T15:41:33.778959001Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.781922211Z","region_metadata_checked_at":"2026-09-18T15:41:33.781922211Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"45a2856401661fdd6bdb02772281f3f65800a3f67401bcdddeefa10e484edbb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"b838f89ca9ff0822f2ef510e9178d1d320d4033a5d5cd3ca5c98537110704efb","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001700260\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-hjgg7zlnkh76\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"name\": \"bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:43:00.850749816Z\",\n \"endTime\": \"2026-09-18T04:12:08.804551345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001700260\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-hjgg7zlnkh76\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"name\": \"bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:43:00.850749816Z\",\n \"endTime\": \"2026-09-18T04:12:08.804551345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a901dd3c6c89997871f3068193d3cb8669ea78d44cff6f15a85a9ea4f76c0503","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"de20ef80e43125ed23aee61f6adcfb8942fa81561feaefb4e80cef19ed4db1c7","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]"}},{"row_id":"dbedf1be4eb49aa3f0b2032cb1641955f889c12ae3f4cb0b2cc59201a7dc52b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"6acf378b1d751cabdd2cb29f20e478c64231e5b033214257ad51d0d59a0c2ba7","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6746a4fee3e6eafc95c01f3459313ede8a21ef2bda7c7dcb1ccbc5b6d468dae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"f081e78216b7bd4a54e652f3dfd3a2cf8cf19638bb76bda01151759533a92078","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100781486385926144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:58:31Z","started_at":"2026-09-18T02:58:31Z","completed_at":"2026-09-18T02:59:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.619823827Z","timing_metadata_checked_at":"2026-09-18T15:41:33.619823827Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:33.622888838Z","region_metadata_checked_at":"2026-09-19T03:30:50.75322187Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8d22f66742ef131f38c7c4e93709705bbe4f7a01bdef73c50caa1c4619b06ca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100781486385926144","occurred_at":"2026-09-18T02:58:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:56:11Z","started_at":"2026-09-18T02:56:11Z","completed_at":"2026-09-18T05:00:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:38.618831126Z","timing_metadata_checked_at":"2026-09-18T15:40:38.618831126Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:38.62260664Z","region_metadata_checked_at":"2026-09-18T15:40:38.62260664Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0fff20b38be4d143d06a2cb3d086f989a5e9bbd8202944d8f44dc5f7fd4f21a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"e2e","signature_id":"c3a814dc2b1fc08b7b4fdb53010472ca317b4e82bd862888bcc25a0f8b4091ea","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-9ptnlfp98gkt/cluster-candidate-4-21-2b86tp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dac60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2b86tp: failed waiting for cluster=\"cluster-candidate-4-21-2b86tp\" in resourcegroup=\"rg-candidate-4-21-9ptnlfp98gkt\" to finish creating: GET https://rp.j2043264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"name\": \"60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:27.058951069Z\",\n \"endTime\": \"2026-09-18T04:15:29.848966802Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-9ptnlfp98gkt/cluster-candidate-4-21-2b86tp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dac60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2b86tp: failed waiting for cluster=\"cluster-candidate-4-21-2b86tp\" in resourcegroup=\"rg-candidate-4-21-9ptnlfp98gkt\" to finish creating: GET https://rp.j2043264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"name\": \"60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:27.058951069Z\",\n \"endTime\": \"2026-09-18T04:15:29.848966802Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"307a452f46841553ddd43cbcf2b027eb5571187d7387898c99ad7aac211d3107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"e2e","signature_id":"66f62a5d5d02e16dfffeec66474c962d8c14552e10473a54125784861158f0d5","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","failure_pattern":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"\u003cvm\u003e\" failed and cleanup also failed: Get \"\u003curl\u003e\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"\u003curl\u003e\": stream error: stream ID 33; CANCEL; received from peer)","search_query":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)"}},{"row_id":"c232fee453456246aa92ab12966ed297d71c38a3433497408526219b347131fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"alert","signature_id":"a0cd1baa14d1b2c8e6eaa3868608b05d0234717e1c141c4d01a43194b1743c96","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T03:58:14Z\n Ended: 2026-09-18T04:08:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2043264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6c4ss9pg11n90barbef44mcqrigek-v20260901\", pod=\"router-85fc7f57f6-7t8xr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6c4ss9pg11n90barbef44mcqrigek-v20260901/router-85fc7f57f6-7t8xr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T04:00:16Z\n Ended: 2026-09-18T04:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2043264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6c4u3st2ot5a3qc2a5p5fm6ceqr6n-t6k8m0v7e7f7n9m\", pod=\"router-5fbdb9d6b6-97rbb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6c4u3st2ot5a3qc2a5p5fm6ceqr6n-t6k8m0v7e7f7n9m/router-5fbdb9d6b6-97rbb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:56:04Z","started_at":"2026-09-18T02:56:04Z","completed_at":"2026-09-18T05:18:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.380449661Z","timing_metadata_checked_at":"2026-09-18T15:40:45.380449661Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.383494872Z","region_metadata_checked_at":"2026-09-18T15:40:45.383494872Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"53b65b43a78d14c7c314b2fc2c97869861ce08c3f4d3ff086f45132c4267d786","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","occurred_at":"2026-09-18T02:56:04Z","failed_at":"e2e","signature_id":"3f4f2a1f835d80221855e8810686dfbc430d2c91ebb857edce8074813fea620c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T02:56:00Z","started_at":"2026-09-18T02:56:00Z","completed_at":"2026-09-18T05:05:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.849291554Z","timing_metadata_checked_at":"2026-09-18T15:40:40.849291554Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.852285665Z","region_metadata_checked_at":"2026-09-18T15:40:40.852285665Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"31f43db8ee5a53149901c5b44af4a35bf237df0e7455875f2880df11b194fafb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"993ed1b03e808393e377627c70564f00855cc6cb2d99bd1882ef2f7b7c73270e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vxdfqwnwx9d5/cluster-hs-pre-wvtqvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001020ba0\u003e: \n failed to create HCP cluster cluster-hs-pre-wvtqvz: failed waiting for cluster=\"cluster-hs-pre-wvtqvz\" in resourcegroup=\"rg-hs-pre--vxdfqwnwx9d5\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"name\": \"28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:19.328892293Z\",\n \"endTime\": \"2026-09-18T04:15:24.561215096Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vxdfqwnwx9d5/cluster-hs-pre-wvtqvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001020ba0\u003e: \n failed to create HCP cluster cluster-hs-pre-wvtqvz: failed waiting for cluster=\"cluster-hs-pre-wvtqvz\" in resourcegroup=\"rg-hs-pre--vxdfqwnwx9d5\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"name\": \"28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:19.328892293Z\",\n \"endTime\": \"2026-09-18T04:15:24.561215096Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4027cb3b82230e7411762c2c3fcf16a5f39c47c82d42c9d2826a972e7806fc24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"a6058215e786bb088f449c6f95b45c11ce369a5297dbcc0542779a89a2c66add","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86e26915f7ac1662975a4eeffe0a1c22227598dded4b96a813ca69614d8e89fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"2e205df2fc76c59d7d5bced445a9af5fdbd8ffc0a4321fbfafdc448b6041ea0f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008963c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6htjbrlxsjhg\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"name\": \"b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:48.434685072Z\",\n \"endTime\": \"2026-09-18T04:15:54.641017549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m9s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008963c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6htjbrlxsjhg\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"name\": \"b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:48.434685072Z\",\n \"endTime\": \"2026-09-18T04:15:54.641017549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m9s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3a38d9eaaacffb8a3810d1f66e0b2e54b8a2e65a0fc51b92dba229525a06db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"a44cbebfe0aee283a67f51149529d8530cd9c5797c013288bce57d67f21ed3fb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dhdrlkcltwdt/cluster-candidate-5-0-tshnff should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d46700\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tshnff: failed waiting for cluster=\"cluster-candidate-5-0-tshnff\" in resourcegroup=\"rg-candidate-5-0-dhdrlkcltwdt\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\",\n \"name\": \"23603031-6375-471f-989c-9b2b7bce844f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:30.045260058Z\",\n \"endTime\": \"2026-09-18T04:15:34.702663145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 13m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dhdrlkcltwdt/cluster-candidate-5-0-tshnff should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d46700\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tshnff: failed waiting for cluster=\"cluster-candidate-5-0-tshnff\" in resourcegroup=\"rg-candidate-5-0-dhdrlkcltwdt\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\",\n \"name\": \"23603031-6375-471f-989c-9b2b7bce844f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:30.045260058Z\",\n \"endTime\": \"2026-09-18T04:15:34.702663145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 13m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a4d1a311e41b5620a81d14547a99e93448ca98bfb8e127aa350cd1bf0b63a0cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T02:55:52Z","started_at":"2026-09-18T02:55:52Z","completed_at":"2026-09-18T05:01:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.447320079Z","timing_metadata_checked_at":"2026-09-18T15:41:21.447320079Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.45019569Z","region_metadata_checked_at":"2026-09-18T15:41:21.45019569Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"80bbad9bae888fa5fe44e9dd4cbac874e35d0034b1f9d3ce998648dbbb165cb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","occurred_at":"2026-09-18T02:55:52Z","failed_at":"e2e","signature_id":"a093a69ee54b8dd613584cc9b286724efd055bbbaf92d5a169de74315234601e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce3a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ql74qfp26s59\" to finish creating: GET https://rp.j7111040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\",\n \"name\": \"0964f917-6834-4736-8825-79c12267a054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:34.744618724Z\",\n \"endTime\": \"2026-09-18T04:17:38.586077059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce3a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ql74qfp26s59\" to finish creating: GET https://rp.j7111040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\",\n \"name\": \"0964f917-6834-4736-8825-79c12267a054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:34.744618724Z\",\n \"endTime\": \"2026-09-18T04:17:38.586077059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780717720997888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:55:28Z","started_at":"2026-09-18T02:55:28Z","completed_at":"2026-09-18T02:55:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.963572155Z","timing_metadata_checked_at":"2026-09-18T15:40:46.963572155Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:46.966884667Z","region_metadata_checked_at":"2026-09-19T03:30:37.884099306Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"98c29963529b7a2a65bb65ecf53acba8e0780b6c1f009068e660ac6a56ba0bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780717720997888","occurred_at":"2026-09-18T02:55:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:51:54Z","started_at":"2026-09-18T02:51:54Z","completed_at":"2026-09-18T04:53:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.49848179Z","timing_metadata_checked_at":"2026-09-18T15:41:33.49848179Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.502061703Z","region_metadata_checked_at":"2026-09-18T15:41:33.502061703Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"bd14580021d6f01bd288a12840fad362aa2a101364cb346e491f26cacec49252","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","occurred_at":"2026-09-18T02:51:54Z","failed_at":"alert","signature_id":"04a95f9ed56a68aa95bb9d5ccdef76442ca4969785b15ebf86b48f429dfe945c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:04:20Z\n Ended: 2026-09-18T04:18:20Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5911552-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5911552-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100777262654689280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"40f10113bb5be6fc421531f63692eacc05043a36","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T02:41:44Z","started_at":"2026-09-18T02:41:44Z","completed_at":"2026-09-18T04:36:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:19.520186622Z","timing_metadata_checked_at":"2026-09-18T15:40:19.520186622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:19.52538674Z","region_metadata_checked_at":"2026-09-18T15:40:19.52538674Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100766002009608192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T01:56:59Z","started_at":"2026-09-18T01:56:59Z","completed_at":"2026-09-18T03:53:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.290367689Z","timing_metadata_checked_at":"2026-09-18T15:41:20.290367689Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.2935147Z","region_metadata_checked_at":"2026-09-18T15:41:20.2935147Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100765928928055296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T01:56:42Z","started_at":"2026-09-18T01:56:42Z","completed_at":"2026-09-18T04:03:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.38469403Z","timing_metadata_checked_at":"2026-09-18T15:40:59.38469403Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.387864841Z","region_metadata_checked_at":"2026-09-18T15:40:59.387864841Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100752869291462656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"45f2172a532769a42d1f99fa0ee222c7efa60b40","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T01:04:48Z","started_at":"2026-09-18T01:04:48Z","completed_at":"2026-09-18T03:06:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.800209557Z","timing_metadata_checked_at":"2026-09-18T15:41:21.800209557Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.803523669Z","region_metadata_checked_at":"2026-09-18T15:41:21.803523669Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c0ebd76b50205ad901e577ade460296f93b4079e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T01:00:47Z","started_at":"2026-09-18T01:00:47Z","completed_at":"2026-09-18T01:30:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.808230852Z","timing_metadata_checked_at":"2026-09-18T15:41:01.808230852Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.811720164Z","region_metadata_checked_at":"2026-09-18T15:41:01.811720164Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"02f23c3c4c01cfeaf268705a12ee578ac8a70edb10cd5df9cb504cbc1f11714c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","occurred_at":"2026-09-18T01:00:47Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-18T01:22:54.255Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T01:22:54.877Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-18T01:22:55.881Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9648512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28\ntime=2026-09-18T01:24:56.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28/operationStatuses/08584119123100595444\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-zhmgezzmxfkbc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j9648512-mgmt-2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T00:42:37Z","started_at":"2026-09-18T00:42:37Z","completed_at":"2026-09-18T02:44:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.359860891Z","timing_metadata_checked_at":"2026-09-18T15:41:33.359860891Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.363504704Z","region_metadata_checked_at":"2026-09-18T15:41:33.363504704Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c1f979c12eb4c64829fb48d994c36008ea43d5333269f88554f4b97ee2c2bc05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","occurred_at":"2026-09-18T00:42:37Z","failed_at":"e2e","signature_id":"82fd789c0354677fc94a578c2ebbcf48d32a60e2464d95dc9709afd5a7216752","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100746534860623872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T00:39:38Z","started_at":"2026-09-18T00:39:38Z","completed_at":"2026-09-18T00:40:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.228241816Z","timing_metadata_checked_at":"2026-09-18T15:41:33.228241816Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:33.231744129Z","region_metadata_checked_at":"2026-09-19T03:30:50.659247726Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f0a4d2a78b6cce5a9956bfb07b3e14edeb5fc1d343d685d099f0eeff8eb2794c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100746534860623872","occurred_at":"2026-09-18T00:39:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":0,"runs_with_occurrences":0,"runs_with_failure_pattern_matches":0,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2101018551178498048","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T18:40:32Z","started_at":"2026-09-18T18:40:32Z","completed_at":"2026-09-18T20:52:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:51.538151743Z","timing_metadata_checked_at":"2026-09-19T03:30:51.538151743Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:51.541772956Z","region_metadata_checked_at":"2026-09-19T03:30:51.541772956Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100915247261945856","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T11:49:58Z","started_at":"2026-09-18T11:49:58Z","completed_at":"2026-09-18T14:13:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:43.236659763Z","timing_metadata_checked_at":"2026-09-18T15:42:43.236659763Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:43.239679874Z","region_metadata_checked_at":"2026-09-18T15:42:43.239679874Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100809166728728576","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T04:48:31Z","started_at":"2026-09-18T04:48:31Z","completed_at":"2026-09-18T07:06:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:43.128554175Z","timing_metadata_checked_at":"2026-09-18T15:42:43.128554175Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:43.131705786Z","region_metadata_checked_at":"2026-09-18T15:42:43.131705786Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":15,"failed_runs":9,"runs_with_occurrences":9,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T23:00:32Z","started_at":"2026-09-18T23:00:32Z","completed_at":"2026-09-19T00:58:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:53.003406006Z","timing_metadata_checked_at":"2026-09-19T03:30:53.003406006Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:53.006325117Z","region_metadata_checked_at":"2026-09-19T03:30:53.006325117Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"259bba84953ec64fd500f1e1cbc63c042bdfbf7e6ef233c9e412c89782c9d5ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"a1f4334212b041c83192bb8549659dd10bbe7f9e8bd239e3cb44f42460834cd6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00114d9e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zgncvtrh7b9x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00114d9e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zgncvtrh7b9x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"280e515396f186e3886a96168105237fdeb9a861ba05abb2a02a44494156bd1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"31f227f2999d078a72461d40e7b0f7d39a61c55e78688c72750e61796cea0c0e","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"424d51ffdd8c2fc3c8452117a476bb9c18f33b5ad1967acfb945ce3cc6ad1e0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"6b756bbe3882dd3ad1f64906dbb7701108fcd8092065edc0fdfc339e155785c9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5fc5a39273ae3815079ff600ab37e72e8590f6e9e89c192c12293185d283ca71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"d656f1256e1df9e299a9ee99092b293d1d1b5f5763759abb016ae0c94e36cbee","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b80720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nz9bqqgzphzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b80720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nz9bqqgzphzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"815f7955493dcfc7a0975aebbdae33ea9d84236a65d7d5081452ba1b848af1f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4e7909db88252003612c7eb675142841e8737860b4601bc7ba1c1c1147d59585","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"88ae6b72fd1057b68a25a91f5d74933c4ed87683bd8c641ef5832bfc4f83f4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"e69d26f721f1d960b9d056d864133ef61ba83a27918de693c75cfc743d5fc4a8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded"}},{"row_id":"c0726cc103e440c504cb6cb2769843c5ae0d5d03a70c2026974c8b0dcf3073f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"19958c8a5897194ef3182c6ab16be4ac9029906aade55bb4256e8a9bbd189d3c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ceb32b2f6ab8ba50124aad0994ad3c53e51b427ca7d9d87166b3dd55ebcb685f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"9ef798cef3d572a0e66143f21817eeeceae70fa379c7c12f948b17be196688fc","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d3581da71ac75f1a969ab2c4b2cefed0e13faba11e644dca136a9992982e31f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"491628b312668f06a0a9bf2464e7de82cea05fd0fafcb43e52e269cbc904606b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d38c28b28de215f24af3b95d1ebf5cb3dd3de972ab46f0d814d3175b8e5fb9a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"8807298fe1e8269d8ba1826e326fe8888ecc37b86d4680f87d9ef650960b1b45","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d7cb3166677cce2b36068777204e054cddad64fdcc620110ae7e7c727911cf22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"889f4b2cac9cd859f4a4017cf24ccfb20aa30ab6ad074847bdec286491246277","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f0d86640b3e51bc4e6e403a2c7ef8cadae28adc5d5b116b53a9d4592784027c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"b07971657c34858950043966384bf3450b2bd8793b71995b336ae5e99ce5d756","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98aa0\u003e: \n failed to create NodePool np-xrmv7w: failed starting nodepool creation \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\" in resourcegroup=\"rg-nightly-4-20-xx4pb26cwd2j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xx4pb26cwd2j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-xrmv7w/nodePools/np-xrmv7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98aa0\u003e: \n failed to create NodePool np-xrmv7w: failed starting nodepool creation \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\" in resourcegroup=\"rg-nightly-4-20-xx4pb26cwd2j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xx4pb26cwd2j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-xrmv7w/nodePools/np-xrmv7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101056312820371456","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T21:10:35Z","started_at":"2026-09-18T21:10:35Z","completed_at":"2026-09-18T23:43:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.726416092Z","timing_metadata_checked_at":"2026-09-19T03:30:52.726416092Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.730859608Z","region_metadata_checked_at":"2026-09-19T03:30:52.730859608Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T17:06:01Z","started_at":"2026-09-18T17:06:01Z","completed_at":"2026-09-18T20:17:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.608720461Z","timing_metadata_checked_at":"2026-09-19T03:30:52.608720461Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.612145174Z","region_metadata_checked_at":"2026-09-19T03:30:52.612145174Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"871c8ec7c1bb43b1a04353558130aef18c8efda71d9f41783a2527f39b69e118","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","occurred_at":"2026-09-18T17:06:01Z","failed_at":"e2e","signature_id":"e22784c3104fac10937a42e7b5069677dd3f893046b62e795e223bfab93b9fcd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003578876366848","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T17:05:15Z","started_at":"2026-09-18T17:05:15Z","completed_at":"2026-09-18T20:38:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.112652046Z","timing_metadata_checked_at":"2026-09-19T03:30:52.112652046Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.115847857Z","region_metadata_checked_at":"2026-09-19T03:30:52.115847857Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T17:02:57Z","started_at":"2026-09-18T17:02:57Z","completed_at":"2026-09-18T20:52:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:51.889272128Z","timing_metadata_checked_at":"2026-09-19T03:30:51.889272128Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:51.892381939Z","region_metadata_checked_at":"2026-09-19T03:30:51.892381939Z"},"failed_at":["e2e"],"failed_test_count":54,"occurrences":[{"row_id":"0211e8e14e2a5b3d4f239b1c4fda1a84aad41a8b73b2fead3f2598f9478c865b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"08737fbd1b615be947880829b89bc4c3a56995f666d4512eff62626a308dd556","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"026d3b03aa916596f1a1e080a00d6204278080d06f2daa4c9aa07de7edf023e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"5bd085aa7d4e7efc364c83356bec393db7595ff945fba164d18f342fedc7d074","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65kllg7cv2w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"name\": \"e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:07.854642874Z\",\n \"endTime\": \"2026-09-18T18:35:13.922823026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65kllg7cv2w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"name\": \"e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:07.854642874Z\",\n \"endTime\": \"2026-09-18T18:35:13.922823026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"028ded3d1d349d230f3014d3133becc8ec61fc7737e04f65d574de6386d0e390","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ed3aaafec71e925920ce96600e6accd52368d8c067bd7c98fb0f3dd54c8990b6","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-w7lg9w\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e2320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-w7lg9w: failed waiting for cluster=\"cp-ystream-upgrade-5-0-w7lg9w\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-whnjtcclp2gm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"name\": \"58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:40.681093019Z\",\n \"endTime\": \"2026-09-18T19:30:40.290116436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-w7lg9w\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e2320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-w7lg9w: failed waiting for cluster=\"cp-ystream-upgrade-5-0-w7lg9w\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-whnjtcclp2gm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"name\": \"58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:40.681093019Z\",\n \"endTime\": \"2026-09-18T19:30:40.290116436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0373f8fa5fb1bd7d61cc75a57fc3b50213c59631360ac5d4f8a1572a4e47d5b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"087e1145946b4175b23e226d2c422f8fbe6adcface041612342c65176f135291","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"155ede202b1081984b4c410c7ba51984193ce8148d01068e8d896ff5ef665dbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"279014bba3c91c8c1147706aa208a7ca8dab431b40a05ae4dc2507ffe4bd82d2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc600\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nhxrjzhdkw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"name\": \"954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:57.113094659Z\",\n \"endTime\": \"2026-09-18T20:23:57.027116398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc600\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nhxrjzhdkw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"name\": \"954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:57.113094659Z\",\n \"endTime\": \"2026-09-18T20:23:57.027116398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"181d3dc35bc76068b616a3506fad1a33714c6daa6635fe953004e37b22d37cb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"020177279fd83a306292e61e3b6eb559ce1bd5ba1e079600d6fe28da054a781b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1edc7ae5a00e1a19fc62643eb83c9ed810818f27e5c0e354f0af6043253c1f1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"7d0f6a2cda1c21acb4480d38818269f535fbffb2494dac6f98b849200db6bc5d","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ef160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v6jpgxxsfw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"name\": \"c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:46.580974029Z\",\n \"endTime\": \"2026-09-18T18:33:53.917669415Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ef160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v6jpgxxsfw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"name\": \"c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:46.580974029Z\",\n \"endTime\": \"2026-09-18T18:33:53.917669415Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"31c14810e54846bacdfc2e4432cab2fbadbbd703197294a85045eb38729d32f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"b971663cd5f630a57997cfad6d7efdb8293b9c2c40b6d6ab7621acde7a5c5511","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-v8gljqqvz6w4/cluster-candidate-4-22-ts6cw9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bebc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ts6cw9: failed waiting for cluster=\"cluster-candidate-4-22-ts6cw9\" in resourcegroup=\"rg-candidate-4-22-v8gljqqvz6w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"name\": \"6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:17.002894711Z\",\n \"endTime\": \"2026-09-18T19:29:20.288780423Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-v8gljqqvz6w4/cluster-candidate-4-22-ts6cw9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bebc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ts6cw9: failed waiting for cluster=\"cluster-candidate-4-22-ts6cw9\" in resourcegroup=\"rg-candidate-4-22-v8gljqqvz6w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"name\": \"6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:17.002894711Z\",\n \"endTime\": \"2026-09-18T19:29:20.288780423Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"331e88311fd0ea4b998a3fdc10cdbfb01eba5a8f61c53f1db7e070c88fc52d6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"bec4f85eb91db9f106f1504bbcfbd5d619f318abcff9dccca5f00cd3af48e378","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-hs7c8h\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204dc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hs7c8h: failed waiting for cluster=\"cluster-zstream-4-20-hs7c8h\" in resourcegroup=\"rg-zstream-upgrade-4-20-4v6msf6jsvhm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"name\": \"fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:45.232517978Z\",\n \"endTime\": \"2026-09-18T19:29:50.287676961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-hs7c8h\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204dc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hs7c8h: failed waiting for cluster=\"cluster-zstream-4-20-hs7c8h\" in resourcegroup=\"rg-zstream-upgrade-4-20-4v6msf6jsvhm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"name\": \"fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:45.232517978Z\",\n \"endTime\": \"2026-09-18T19:29:50.287676961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37d6c057da66b2131f940542ca522868cfd86f188b450106747c480a75abe258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"bfbf541997c9dcf2457e39f6d9a3f3675b64df774c7c81e642e24bee8b14665a","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2e60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-92vpld7m4q55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"name\": \"043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.023266499Z\",\n \"endTime\": \"2026-09-18T20:24:27.026850238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2e60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-92vpld7m4q55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"name\": \"043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.023266499Z\",\n \"endTime\": \"2026-09-18T20:24:27.026850238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37f66f6788749252764b0a8c2c2f1bda1581ac4c5605743f5364bdc2196fcde7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"60ea4497c6186b3e803d7e6ac044255c8c915fd98b1ff264e2058f0d46b0df0e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded"}},{"row_id":"46490c8efaf59582c890fd95c754902cffd4813cff9e0645e96637af76d19cb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"48193ab97021b80941212fe6bcda29d0c742e88d6b74bd22b1e5757b7e1b3bbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"88f393269cef98e4be13d3c8ab69c7daf8948bb76a2277ef306df923eea104ad","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000872e40\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9pzlkssjh8rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"name\": \"32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:49.139622647Z\",\n \"endTime\": \"2026-09-18T19:28:50.286578702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000872e40\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9pzlkssjh8rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"name\": \"32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:49.139622647Z\",\n \"endTime\": \"2026-09-18T19:28:50.286578702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d1e66fffc20ec5e58eecd30aebd40c1a11ebb910dd534010855e07e3b8c59d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"d992a344dec95549926180b213136a61987bad3b2b2cfe5a8cedabfb0cba64d2","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000730dc0\u003e: \n failed to create HCP cluster ea-listht4pz6: failed waiting for cluster=\"ea-listht4pz6\" in resourcegroup=\"ea-list-zfzjn95sp6z4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"name\": \"1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:22.470708216Z\",\n \"endTime\": \"2026-09-18T19:31:30.292183292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000730dc0\u003e: \n failed to create HCP cluster ea-listht4pz6: failed waiting for cluster=\"ea-listht4pz6\" in resourcegroup=\"ea-list-zfzjn95sp6z4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"name\": \"1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:22.470708216Z\",\n \"endTime\": \"2026-09-18T19:31:30.292183292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4fd60e4df1936e8425e6757a96602bd1361a328d7fa10f97e72f1f1f7973dfde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8c17a49c3b1087a5252885b16d63b8887ba96a24c838b70dae74c83a59595562","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e460\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gq5kj4t7v9lj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"name\": \"33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:54.936239942Z\",\n \"endTime\": \"2026-09-18T18:35:03.923586817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e460\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gq5kj4t7v9lj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"name\": \"33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:54.936239942Z\",\n \"endTime\": \"2026-09-18T18:35:03.923586817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5394b36300091ca0c568cba92d1ad99ffca033e0f84d929b305fd4d4cf235239","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"23eff0283d14160643a2c1b951b106f62c524eb771d25036e9a90406a0924c53","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e7760\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-krl8lpfgmlzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"name\": \"4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.707548645Z\",\n \"endTime\": \"2026-09-18T18:33:33.918055388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e7760\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-krl8lpfgmlzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"name\": \"4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.707548645Z\",\n \"endTime\": \"2026-09-18T18:33:33.918055388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"58323156d12d6e1d9e17febc1c8b442b6542dca5f880dc95ebb22d804fd4eb44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"cb28d0b51212f011a6a9a6a3530556dc58c4a4d691d64d362e422cfc091e42ee","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077bdc0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsw82p5n5s4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"name\": \"5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:11.953070547Z\",\n \"endTime\": \"2026-09-18T19:31:20.290227685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077bdc0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsw82p5n5s4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"name\": \"5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:11.953070547Z\",\n \"endTime\": \"2026-09-18T19:31:20.290227685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"607499447b8a29de495eb18348a9a9d13e0639eea66ced853e22cfe011e86743","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"18459d5e026c91320fea1e6f4619d65e0f3c2cce677a7bf1890dbe9529aa82dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vnd6pm4rt5md/cluster-candidate-4-21-gkgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gkgzcb: failed waiting for cluster=\"cluster-candidate-4-21-gkgzcb\" in resourcegroup=\"rg-candidate-4-21-vnd6pm4rt5md\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"name\": \"fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.815434003Z\",\n \"endTime\": \"2026-09-18T19:29:20.286956387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vnd6pm4rt5md/cluster-candidate-4-21-gkgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gkgzcb: failed waiting for cluster=\"cluster-candidate-4-21-gkgzcb\" in resourcegroup=\"rg-candidate-4-21-vnd6pm4rt5md\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"name\": \"fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.815434003Z\",\n \"endTime\": \"2026-09-18T19:29:20.286956387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"621b83b49d4450c11472cfce62b9a69bfaf21557764758dd59b3472d160e4364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"4d196dbdd6adc5cbd0b42174989b35c61699ec802a9b43e61050ca43a4a526db","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2c500\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-djhd95nkq4ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"name\": \"f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:48.39283405Z\",\n \"endTime\": \"2026-09-18T20:26:57.037154264Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2c500\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-djhd95nkq4ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"name\": \"f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:48.39283405Z\",\n \"endTime\": \"2026-09-18T20:26:57.037154264Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"691e134cb1e2fe5b7b0440dfbd28efa2a597a50cae196ce8be46b9c1a5bb18d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"f6a9a5ad677f486ebe2e2116b48d59eff2dab4dc02f9785191be923be7559549","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314da0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rgw6mwjd2jdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"name\": \"09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:35.503440276Z\",\n \"endTime\": \"2026-09-18T18:33:43.918161334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314da0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rgw6mwjd2jdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"name\": \"09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:35.503440276Z\",\n \"endTime\": \"2026-09-18T18:33:43.918161334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ac49c323b67c7d7110ee9bd50fbf775e842e6aecd22499e02812121d7a59453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"a086d8d16ebbda1f55085340c84766813ece581729f4e4db8f61f9c0b1339c8e","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010724a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-fp6bsqq92kn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"name\": \"55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.41327514Z\",\n \"endTime\": \"2026-09-18T18:33:23.916940698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010724a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-fp6bsqq92kn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"name\": \"55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.41327514Z\",\n \"endTime\": \"2026-09-18T18:33:23.916940698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c9e225debfb06a93ce8fd4a4082d4065a29742d6d896646708869494b7fb8aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"807a30302c1afe55628a0e91c1d8d2d1cf846d02300aeafd38841fdc5629959a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-cjg6sk\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ed60\u003e: \n failed to create HCP cluster cluster-zstream-4-21-cjg6sk: failed waiting for cluster=\"cluster-zstream-4-21-cjg6sk\" in resourcegroup=\"rg-zstream-upgrade-4-21-h8qwfrzsn2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"name\": \"e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:38.211424873Z\",\n \"endTime\": \"2026-09-18T19:29:40.287386024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-cjg6sk\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ed60\u003e: \n failed to create HCP cluster cluster-zstream-4-21-cjg6sk: failed waiting for cluster=\"cluster-zstream-4-21-cjg6sk\" in resourcegroup=\"rg-zstream-upgrade-4-21-h8qwfrzsn2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"name\": \"e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:38.211424873Z\",\n \"endTime\": \"2026-09-18T19:29:40.287386024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75713d57b69945b925dcf8e93e672f871273df96518a658f44ae78319e9b2052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"3a36b2cb0e45db6b1b5adf537bf6a65a0dc2daa7ebc74fcf104789a0636dc715","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107a4a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nsq6nncs9jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"name\": \"d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:51.185500713Z\",\n \"endTime\": \"2026-09-18T20:25:57.031163299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107a4a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nsq6nncs9jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"name\": \"d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:51.185500713Z\",\n \"endTime\": \"2026-09-18T20:25:57.031163299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"761d0fa8553ef7bd44b1a22217c6da419a24c3dee00e22f2ceb7581c1d766865","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2c5047fbc800377676e3159b01faa4b14e97abaaef72bb1d0f7c177a6a8e963a","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ca440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hx4h2xzxb7kg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"name\": \"e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:09.060511648Z\",\n \"endTime\": \"2026-09-18T20:24:17.026518831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ca440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hx4h2xzxb7kg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"name\": \"e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:09.060511648Z\",\n \"endTime\": \"2026-09-18T20:24:17.026518831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bf88482afa590949a19c3f9fff5eb3667ed0d7458569a1b3ce7d0458a14ab51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ac027a2fae7f9c119c5d13e433aca363beeb537c7f13236bce5b4509168d72e7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c320\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ntmzhdz9dn8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"name\": \"304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:03.328284727Z\",\n \"endTime\": \"2026-09-18T20:26:07.032768851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c320\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ntmzhdz9dn8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"name\": \"304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:03.328284727Z\",\n \"endTime\": \"2026-09-18T20:26:07.032768851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80a4ddf1a1a99f8ee68dfb28a5326f3b971ff20de102e0de13972ae881f86611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8f8a4a3e3e075b660c1fb0534694eabcbec0ee97ddf1c7de50ae027a7c989e5e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e42a80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x6ksv4qd5hh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\",\n \"name\": \"0d11158a-c610-4ba0-b070-6f011685df27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.656789631Z\",\n \"endTime\": \"2026-09-18T18:35:23.922596176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e42a80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x6ksv4qd5hh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\",\n \"name\": \"0d11158a-c610-4ba0-b070-6f011685df27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.656789631Z\",\n \"endTime\": \"2026-09-18T18:35:23.922596176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80ab17b227e56f09e06a80ab71422ddb33e0236760a76a4c1b70f1688e33f9b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"55f11975675616706103b6b7daff6a4b423e0747ce59c5e7da66f3dc349981e7","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e4e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5l7s7j7vcxkh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"name\": \"6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:09.380870167Z\",\n \"endTime\": \"2026-09-18T19:31:10.29248977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e4e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5l7s7j7vcxkh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"name\": \"6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:09.380870167Z\",\n \"endTime\": \"2026-09-18T19:31:10.29248977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"81c7d6e579d2227b7a4c6ea47e6755cf7335f6d46baea2ddaa010b066e9a2744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ec2f8559597c2e2e96353159ad3c18cb0e7f4c1ae2d3f60c226c0178a7ef5d2b","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab4bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9x5xbv4nzw8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"name\": \"33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:03.940822361Z\",\n \"endTime\": \"2026-09-18T20:24:07.0250673Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab4bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9x5xbv4nzw8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"name\": \"33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:03.940822361Z\",\n \"endTime\": \"2026-09-18T20:24:07.0250673Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84ac497f7e7f99cb1175249dc1d48e5993ac88dda3cbe590d6a110469fcb8764","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2bd526e04864e297297108f31c386e629877109b67a0cd5577f66e966f4820a0","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90f60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-z57bbtc664tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\",\n \"name\": \"5f403839-210b-4649-acab-f3cebf6c199e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:56.042833825Z\",\n \"endTime\": \"2026-09-18T19:29:00.285060799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90f60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-z57bbtc664tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\",\n \"name\": \"5f403839-210b-4649-acab-f3cebf6c199e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:56.042833825Z\",\n \"endTime\": \"2026-09-18T19:29:00.285060799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89ad59949d4cfe54b57f74b4db11d3f769ddaa5807f73b34f6c87be8fb46edbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"18cde222ab56ffb759e053d76ee6e19c739c516c2db2d311138d345c27569034","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e17a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qcdwg6ktcch6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"name\": \"b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:24.536461143Z\",\n \"endTime\": \"2026-09-18T18:33:33.916870368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e17a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qcdwg6ktcch6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"name\": \"b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:24.536461143Z\",\n \"endTime\": \"2026-09-18T18:33:33.916870368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c00e4952c81686effa8b71a69a711a09892789e24e73cc03601bd75fafd09e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"8f472cc83c135324ad24ab195d238f2ea80f11a76ff49c1665f921d0f019a309","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"300283533309312abfaf8eb06c5bc243249dba952ac99795631a8cca96c1b04d","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9ab80\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qp94zrf9dxjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"name\": \"1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:31.378329777Z\",\n \"endTime\": \"2026-09-18T20:25:37.031824413Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9ab80\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qp94zrf9dxjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"name\": \"1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:31.378329777Z\",\n \"endTime\": \"2026-09-18T20:25:37.031824413Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97bdc90c8a1291bc689a1b501c7b566934f460c4a1691ce224e4affe9cacea64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"eabc0e9f060f2e7a3e41ea442762aca89d3441bef1655db8dca9669defa6ff8c","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6480\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p4nctxb27lkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\",\n \"name\": \"47247d9a-d966-4134-b361-76bb41af047f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.12537559Z\",\n \"endTime\": \"2026-09-18T18:33:23.915581975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6480\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p4nctxb27lkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\",\n \"name\": \"47247d9a-d966-4134-b361-76bb41af047f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.12537559Z\",\n \"endTime\": \"2026-09-18T18:33:23.915581975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9904101b08db6937d4497fde1a011de06201d82d8d83d070b527ab492f051cfe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"980439aafe10d6730454288b03e5c4b325ad4d9a23209ccce9ea0925ca8727b5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-njqww7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af10c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-njqww7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-njqww7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jpqnx2lrkvjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"name\": \"b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:31.270212929Z\",\n \"endTime\": \"2026-09-18T19:30:40.29151616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-njqww7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af10c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-njqww7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-njqww7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jpqnx2lrkvjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"name\": \"b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:31.270212929Z\",\n \"endTime\": \"2026-09-18T19:30:40.29151616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b629ae8802b9eee0233f7dfc71c223fef82765ee22d82d8817a0a40fde6f397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"aba4ff8b5edea60411faee257f0fcca1514022f58c987167a980974659ac599f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b0380\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8dhk5572s4k2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"name\": \"0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:07:16.233683149Z\",\n \"endTime\": \"2026-09-18T18:36:20.211533932Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b0380\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8dhk5572s4k2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"name\": \"0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:07:16.233683149Z\",\n \"endTime\": \"2026-09-18T18:36:20.211533932Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b6d1e1e893a32cbbdaf40f8ebcdec1d69bd66b92f9cc8bac39a913c0b69bb22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2a4e0071788107d1b0f28cafcae777b74d59c2ed92ca22e3caeb7564b1685be1","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14ec0\u003e: \n failed to create HCP cluster np-dg-minor-xwhdp5: failed waiting for cluster=\"np-dg-minor-xwhdp5\" in resourcegroup=\"rg-np-dg-minor-xwhdp5-592m9fw6ltl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"name\": \"35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:39.90066671Z\",\n \"endTime\": \"2026-09-18T20:25:47.031092558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14ec0\u003e: \n failed to create HCP cluster np-dg-minor-xwhdp5: failed waiting for cluster=\"np-dg-minor-xwhdp5\" in resourcegroup=\"rg-np-dg-minor-xwhdp5-592m9fw6ltl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"name\": \"35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:39.90066671Z\",\n \"endTime\": \"2026-09-18T20:25:47.031092558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a0f076a5c11daca409b4a8602d8e4ce9f653ba31dad49cecc3563150f920d98f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"9da2d69669521624ba370e29cdccd478000fd77fa5acf531ed0100da09e598f1","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010da5a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6ppzp5frfmgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"name\": \"18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:25.031159118Z\",\n \"endTime\": \"2026-09-18T18:33:33.919612215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010da5a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6ppzp5frfmgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"name\": \"18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:25.031159118Z\",\n \"endTime\": \"2026-09-18T18:33:33.919612215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6575061b5d12438545743324e8367c9fea437febcee658a82868bc41212d5b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"12c88ab455c66941eb056387e08ab482c4399c448c1844b1db1d64ca2a21923b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-7ggpqb\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-7ggpqb: failed waiting for cluster=\"cluster-zstream-4-22-7ggpqb\" in resourcegroup=\"rg-zstream-upgrade-4-22-nfmkkc97chvl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"name\": \"14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.092484844Z\",\n \"endTime\": \"2026-09-18T19:29:40.290112371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-7ggpqb\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-7ggpqb: failed waiting for cluster=\"cluster-zstream-4-22-7ggpqb\" in resourcegroup=\"rg-zstream-upgrade-4-22-nfmkkc97chvl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"name\": \"14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.092484844Z\",\n \"endTime\": \"2026-09-18T19:29:40.290112371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a82052cd828fbfd99b59f191e641dc7ad54d047f6e611710007c17f8f2b6e9f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"99a0fc3be3299d125a1312f83223671d90940dffaa62fee2e5b18166cfca67f4","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011faaa0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-smxqhhn278sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"name\": \"ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.39543422Z\",\n \"endTime\": \"2026-09-18T18:33:23.917604609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011faaa0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-smxqhhn278sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"name\": \"ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.39543422Z\",\n \"endTime\": \"2026-09-18T18:33:23.917604609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9350470812b02e997b9125bc024f29804cb288e2de4809951bb8e962e38780f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"c9577e8c1b375d41d27d0e18df18642cc9e2a302455c27112af6cdf04b27da87","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kg5zmv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfa6c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kg5zmv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-kg5zmv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gsppncf9n78j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"name\": \"6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:34.659094453Z\",\n \"endTime\": \"2026-09-18T19:30:40.290599044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kg5zmv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfa6c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kg5zmv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-kg5zmv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gsppncf9n78j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"name\": \"6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:34.659094453Z\",\n \"endTime\": \"2026-09-18T19:30:40.290599044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b96757dbe2b79bd77de7e95dbc6b4fea759516a80e61fa6e5ab2ddd4a2b3e109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"596b0ffa1cf24f21e4c89da0dd1bf8032daa39c9dfe890cbc997391f5e16d758","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcf3c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2v27j8w6flzw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"name\": \"3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:31.190318654Z\",\n \"endTime\": \"2026-09-18T18:33:33.918230691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcf3c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2v27j8w6flzw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"name\": \"3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:31.190318654Z\",\n \"endTime\": \"2026-09-18T18:33:33.918230691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cab1f907aa7e25850325ac850e1a2c316e4a3b3bbb5412886085dfe492af969f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"d9b87f83858caf3b228ef9b132bcd625c28601b6944272225765d7e4212bfbd4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jspqshnm6b8s/cluster-candidate-4-20-26p7kj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-26p7kj: failed waiting for cluster=\"cluster-candidate-4-20-26p7kj\" in resourcegroup=\"rg-candidate-4-20-jspqshnm6b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"name\": \"670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.619385936Z\",\n \"endTime\": \"2026-09-18T19:29:20.288515017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jspqshnm6b8s/cluster-candidate-4-20-26p7kj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-26p7kj: failed waiting for cluster=\"cluster-candidate-4-20-26p7kj\" in resourcegroup=\"rg-candidate-4-20-jspqshnm6b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"name\": \"670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.619385936Z\",\n \"endTime\": \"2026-09-18T19:29:20.288515017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd20643ec408ce450bac1a54eba04042b26f3d3b640fbdbe5714cf8d7cec9897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"fcaef2f74196ab4f30c8d92bee5653f6add69a944685179ec76976f1aebee54b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6d540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc84t6g5h54p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"name\": \"33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.507928047Z\",\n \"endTime\": \"2026-09-18T18:33:33.91760498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6d540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc84t6g5h54p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"name\": \"33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.507928047Z\",\n \"endTime\": \"2026-09-18T18:33:33.91760498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd2c2a65757d4315af35d186835f3bcfa5bf77df9289f64d8aca90b0c3fcfb80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"cf464e2390b41d88251c211ae76f31739afd822d6d94414c2b9fbfc2f709feaa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pc4nmpfmfd8n/cluster-candidate-5-0-8k9kgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008977c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8k9kgf: failed waiting for cluster=\"cluster-candidate-5-0-8k9kgf\" in resourcegroup=\"rg-candidate-5-0-pc4nmpfmfd8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\",\n \"name\": \"b151a448-87ca-4125-a955-ff96a58ce701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:04.863718426Z\",\n \"endTime\": \"2026-09-18T19:29:10.287329661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pc4nmpfmfd8n/cluster-candidate-5-0-8k9kgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008977c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8k9kgf: failed waiting for cluster=\"cluster-candidate-5-0-8k9kgf\" in resourcegroup=\"rg-candidate-5-0-pc4nmpfmfd8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\",\n \"name\": \"b151a448-87ca-4125-a955-ff96a58ce701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:04.863718426Z\",\n \"endTime\": \"2026-09-18T19:29:10.287329661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cdde3d4c4687481087e9c788eb5069c6bdb0503fa226dba008b4f89e89f93ff6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"f52d9598c009df2683fa8e80fa13ffac13c236506f242fa3de8742e288cc9518","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-tpqdn5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e760\u003e: \n failed to create HCP cluster np-skip-minor-tpqdn5: failed waiting for cluster=\"np-skip-minor-tpqdn5\" in resourcegroup=\"rg-np-skip-minor-tpqdn5-tc97l4b65b5k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"name\": \"06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:52.428957278Z\",\n \"endTime\": \"2026-09-18T20:24:57.029688909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-tpqdn5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e760\u003e: \n failed to create HCP cluster np-skip-minor-tpqdn5: failed waiting for cluster=\"np-skip-minor-tpqdn5\" in resourcegroup=\"rg-np-skip-minor-tpqdn5-tc97l4b65b5k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"name\": \"06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:52.428957278Z\",\n \"endTime\": \"2026-09-18T20:24:57.029688909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ced8f4053252ac14c6f638a3a6f16d54244333fd644369ee54b48a86cdb7cc82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"612c1b0bfa8c63e918534f55b46917bda5292bb668f43f5109397135c5f16839","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wlmbkw\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24ce0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wlmbkw: failed waiting for cluster=\"cluster-zstream-5-0-wlmbkw\" in resourcegroup=\"rg-zstream-upgrade-5-0-88f465g4v8qg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"name\": \"8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.645326719Z\",\n \"endTime\": \"2026-09-18T19:29:40.288068036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wlmbkw\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24ce0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wlmbkw: failed waiting for cluster=\"cluster-zstream-5-0-wlmbkw\" in resourcegroup=\"rg-zstream-upgrade-5-0-88f465g4v8qg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"name\": \"8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.645326719Z\",\n \"endTime\": \"2026-09-18T19:29:40.288068036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d40139f7ef37bcb47627e083dba680a368db24cbb9aa90f6514b6709ddaa3f6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8e3503c7dac56f3fd400f32d8b3f5b71ecf5acd0c55bc2d5e52b0b130348d093","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-phwsxm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc9c0\u003e: \n failed to create HCP cluster np-noedge-phwsxm: failed waiting for cluster=\"np-noedge-phwsxm\" in resourcegroup=\"rg-np-noedge-phwsxm-4bv8jxxfz58h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"name\": \"ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:33.085250323Z\",\n \"endTime\": \"2026-09-18T20:24:37.028572057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-phwsxm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc9c0\u003e: \n failed to create HCP cluster np-noedge-phwsxm: failed waiting for cluster=\"np-noedge-phwsxm\" in resourcegroup=\"rg-np-noedge-phwsxm-4bv8jxxfz58h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"name\": \"ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:33.085250323Z\",\n \"endTime\": \"2026-09-18T20:24:37.028572057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e317527bcfba5b41113f4d3f3df8dfb232d83b2119b8223f2b165223b4571a5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"b09c9c83b49154876ea9edaee019cac3ced0de0e27ddbe6889ee50f6b423f2ad","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld4s6tgxr5vg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"name\": \"a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:16:16.855336614Z\",\n \"endTime\": \"2026-09-18T19:45:20.354805698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld4s6tgxr5vg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"name\": \"a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:16:16.855336614Z\",\n \"endTime\": \"2026-09-18T19:45:20.354805698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ead2169eefc3a2c8aec7cc6799696a576e5414661e3b0a307a5b0b669d06504c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"55df98f2e805e8d535825529f230ec77e5caf64b117f366392092a2ef0f04238","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c425a0\u003e: \n failed to create HCP cluster np-downgrade-7d26bl: failed waiting for cluster=\"np-downgrade-7d26bl\" in resourcegroup=\"rg-np-downgrade-7d26bl-7qmvc9mdqdmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"name\": \"02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:45.411002197Z\",\n \"endTime\": \"2026-09-18T20:24:47.028298292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c425a0\u003e: \n failed to create HCP cluster np-downgrade-7d26bl: failed waiting for cluster=\"np-downgrade-7d26bl\" in resourcegroup=\"rg-np-downgrade-7d26bl-7qmvc9mdqdmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"name\": \"02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:45.411002197Z\",\n \"endTime\": \"2026-09-18T20:24:47.028298292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec56c0b165a44db2deaa54ac9c25317f315fa48b0d4c8a782cfd702c07e6fe87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"c71f8c34107674e9b83007fd4c4382acbf933644d3d70e04b10c2bce3d213437","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011ed1a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"name\": \"d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:57.828166643Z\",\n \"endTime\": \"2026-09-18T18:35:03.924193327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011ed1a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"name\": \"d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:57.828166643Z\",\n \"endTime\": \"2026-09-18T18:35:03.924193327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0a818c35c7b74c13a9a05f8d18ada59065ca609029af910ebce486e547b9549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"9253c3493335b5dfd398622bbe3451ad8bddf74417d1a8c0037aece178838cde","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-59kkxj with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90940\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-59kkxj: failed waiting for cluster=\"np-version-upgrade-cluster-59kkxj\" in resourcegroup=\"rg-np-version-upgrade-59kkxj-strjcq5cf28t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"name\": \"c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:19.582015951Z\",\n \"endTime\": \"2026-09-18T20:24:27.027192644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-59kkxj with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90940\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-59kkxj: failed waiting for cluster=\"np-version-upgrade-cluster-59kkxj\" in resourcegroup=\"rg-np-version-upgrade-59kkxj-strjcq5cf28t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"name\": \"c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:19.582015951Z\",\n \"endTime\": \"2026-09-18T20:24:27.027192644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f403f49fafa7eaba741dba44bdfbac7a3fcb424d950c14ac38fc7bc11208982c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"19596332cc8edb7c2fe7ff05b62421eeeea53668cb594d6f172e695930cfc93e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016acc40\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t8z82lzs48sr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"name\": \"ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.431481456Z\",\n \"endTime\": \"2026-09-18T18:33:33.917230374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016acc40\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t8z82lzs48sr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"name\": \"ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.431481456Z\",\n \"endTime\": \"2026-09-18T18:33:33.917230374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f457631bf2de98a8eb53777c8928d27f0e7c666d176e4ab60a495aafbeb039f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2e5039616cb1c8d7f4ac14c50b2a06f1fe40081131c5acd543f28d2cc514abec","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015aed60\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vxq45g66n2fb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"name\": \"1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:43.251936528Z\",\n \"endTime\": \"2026-09-18T19:28:50.284453068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015aed60\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vxq45g66n2fb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"name\": \"1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:43.251936528Z\",\n \"endTime\": \"2026-09-18T19:28:50.284453068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f721de984f1e9d2adc21d569363335cf38d2031abf0e310bc9cec2d464a4c943","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ccebff601285f1ea3ac04ae7a355282a8c0e14e4f8eb582eebc840c11f6d9a08","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gs8fc2 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbc920\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gs8fc2: failed waiting for cluster=\"np-version-upgrade-cluster-gs8fc2\" in resourcegroup=\"rg-np-version-upgrade-gs8fc2-flflx6nnvpbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\",\n \"name\": \"602c1354-35fa-4664-9d86-220f8f644a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:14.360447552Z\",\n \"endTime\": \"2026-09-18T20:24:17.027918855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gs8fc2 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbc920\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gs8fc2: failed waiting for cluster=\"np-version-upgrade-cluster-gs8fc2\" in resourcegroup=\"rg-np-version-upgrade-gs8fc2-flflx6nnvpbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\",\n \"name\": \"602c1354-35fa-4664-9d86-220f8f644a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:14.360447552Z\",\n \"endTime\": \"2026-09-18T20:24:17.027918855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":54,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7"],"matched_occurrences":54,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939709793439744","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:27:14Z","started_at":"2026-09-18T13:27:14Z","completed_at":"2026-09-18T16:03:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:11:55.679083178Z","timing_metadata_checked_at":"2026-09-18T16:11:55.679083178Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:11:55.683053992Z","region_metadata_checked_at":"2026-09-18T16:11:55.683053992Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939566205636608","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:26:34Z","started_at":"2026-09-18T13:26:34Z","completed_at":"2026-09-18T15:54:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:01:33.186908727Z","timing_metadata_checked_at":"2026-09-18T16:01:33.186908727Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:01:33.19001214Z","region_metadata_checked_at":"2026-09-18T16:01:33.19001214Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939394377584640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:25:59Z","started_at":"2026-09-18T13:25:59Z","completed_at":"2026-09-18T15:57:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:01:33.059102212Z","timing_metadata_checked_at":"2026-09-18T16:01:33.059102212Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:01:33.065093236Z","region_metadata_checked_at":"2026-09-18T16:01:33.065093236Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:26:04Z","started_at":"2026-09-18T10:26:04Z","completed_at":"2026-09-18T12:54:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.688508858Z","timing_metadata_checked_at":"2026-09-18T15:42:48.688508858Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.691603169Z","region_metadata_checked_at":"2026-09-18T15:42:48.691603169Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7a16b7d73638182fb5e8d36595d4f82bd635e5aa75a7038b4e6eb77adc46261e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","occurred_at":"2026-09-18T10:26:04Z","failed_at":"e2e","signature_id":"3269dcb6be86d6a6047ecae085ca193c45aec85e3774519113b1593c115dfa87","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:20:35Z","started_at":"2026-09-18T10:20:35Z","completed_at":"2026-09-18T12:47:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.548336754Z","timing_metadata_checked_at":"2026-09-18T15:42:48.548336754Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.551590466Z","region_metadata_checked_at":"2026-09-18T15:42:48.551590466Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0fb792a233958dd2d09e4c8ba1ccd3cc8b8e162437215b1cf4005751b59010f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","occurred_at":"2026-09-18T10:20:35Z","failed_at":"e2e","signature_id":"d7f2aa1ec90b6da0508d97952c26f5e68a44fec6becea93e11a0c8acd2ecf80a","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100891307781132288","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T10:14:55Z","started_at":"2026-09-18T10:14:55Z","completed_at":"2026-09-18T12:50:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.421816799Z","timing_metadata_checked_at":"2026-09-18T15:42:48.421816799Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.431526734Z","region_metadata_checked_at":"2026-09-18T15:42:48.431526734Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:10:35Z","started_at":"2026-09-18T10:10:35Z","completed_at":"2026-09-18T12:36:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.204624418Z","timing_metadata_checked_at":"2026-09-18T15:42:48.204624418Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.20782373Z","region_metadata_checked_at":"2026-09-18T15:42:48.20782373Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c257c9cf8f6ee1561328a5d8652a6d4630c8c215eb8d7deb94b19d07bff1dbfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","occurred_at":"2026-09-18T10:10:35Z","failed_at":"e2e","signature_id":"5533b31e4276757a4afc74d1ad67148e45245ccef8ea260d65b9e08077693a1d","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:58:50Z","started_at":"2026-09-18T05:58:50Z","completed_at":"2026-09-18T08:28:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.992752057Z","timing_metadata_checked_at":"2026-09-18T15:42:47.992752057Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.995942368Z","region_metadata_checked_at":"2026-09-18T15:42:47.995942368Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cd86f0ae9a6585720a683c4ec931484238a8d28bf0831d63a340257ec5268052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","occurred_at":"2026-09-18T05:58:50Z","failed_at":"e2e","signature_id":"7cb7501ef4b819fdc17f5b5e3ca2619d486971f5a3c25a5533b8ff273a412172","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:00:17Z","started_at":"2026-09-18T02:00:17Z","completed_at":"2026-09-18T04:19:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:50.478297492Z","timing_metadata_checked_at":"2026-09-18T15:42:50.478297492Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:50.482649407Z","region_metadata_checked_at":"2026-09-18T15:42:50.482649407Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"442af769c0c7e3c0a8fb59afc4f8ad6ca5129510396948fa20c1e16b0ee3f399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"6e4745afece375c35c57231f904892210b15b74988a6f2baceda810fa43da8e5","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4436f72f86dcdf7d5b39b762f07fd973095e73e991a63abf4213f7c8428ef204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"60cc795ca9532d84d0a6a8c864d8b05a2f5285c1bfd6c8aa612a88d8890d8bb5","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"953a178f8aa729db097bdc70aca88aeda09ff4fcc8e7558bccc494ea7e775e80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"6fd32498c414e944f0c010bd303fd297e25f438aa938164d7b89331e4fe1bf3e","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012e0990\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012e0990\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}},{"row_id":"d3810f538bd4859e129bbd454ece2c7089a38c3720080d94e45f571849456219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"8be7c23a1e56f277787ec21fa08eb46e46af4b8580d42385a8b71372839091d6","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd20\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-hg2tcz\" in resourcegroup=\"rg-np-dg-minor-hg2tcz-n7pnlmn2w4pv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"name\": \"bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:32:13.01856929Z\",\n \"endTime\": \"2026-09-18T03:51:16.292258995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd20\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-hg2tcz\" in resourcegroup=\"rg-np-dg-minor-hg2tcz-n7pnlmn2w4pv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"name\": \"bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:32:13.01856929Z\",\n \"endTime\": \"2026-09-18T03:51:16.292258995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T01:44:31Z","started_at":"2026-09-18T01:44:31Z","completed_at":"2026-09-18T04:25:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.858222373Z","timing_metadata_checked_at":"2026-09-18T15:42:47.858222373Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.861839386Z","region_metadata_checked_at":"2026-09-18T15:42:47.861839386Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"15361385a7407ab14f0a513e66c95ec47f5e75567c7e48cc4eaa99de087ec84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"736f2b0814fabfc59b7e462d9755a0e748e4d396c2743f81540491fd582dab16","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39025eae0aa723d1be917bb4455954c57ec535d64f978439377e18b5ca3efe61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"f164ce20210c5901433b68d8b32c7bafd9db9fbdb4b9f088ddfe4a53eb8ff36e","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001176ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001176ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}},{"row_id":"ee0ca8706b64a81e3310ae86225db08f16064fdc710fab225376bf191551ae61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"812ff7352462ae5b2ffc2a98177a772e72a7a199983bda3af2609cfd08e8cb77","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005b20\u003e: \n failed to create NodePool np-pd9r72: failed waiting for nodepool=\"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\" in resourcegroup=\"rg-candidate-4-20-7nvxhnmpb6qr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"name\": \"e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:55:35.193527856Z\",\n \"endTime\": \"2026-09-18T03:14:42.13127378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005b20\u003e: \n failed to create NodePool np-pd9r72: failed waiting for nodepool=\"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\" in resourcegroup=\"rg-candidate-4-20-7nvxhnmpb6qr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"name\": \"e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:55:35.193527856Z\",\n \"endTime\": \"2026-09-18T03:14:42.13127378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T23:00:32Z","started_at":"2026-09-18T23:00:32Z","completed_at":"2026-09-18T23:16:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:53.229884935Z","timing_metadata_checked_at":"2026-09-19T03:30:53.229884935Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:53.233203047Z","region_metadata_checked_at":"2026-09-19T03:30:53.233203047Z"},"failed_at":["e2e"],"failed_test_count":52,"occurrences":[{"row_id":"00459e24a1bc2f11ee1fe98e4eeeb5af2dc8ea335ef3c4b79cdd746e6b2662be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"e5ac7068f8a4f91ce6b642a6a37bd31dd8ace822a1805ae8325512e343c782e9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-6cpntrp4snvc/cluster-nightly-4-23-t8pvmw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00039c500\u003e: \n failed to create HCP cluster cluster-nightly-4-23-t8pvmw: failed starting cluster creation \"cluster-nightly-4-23-t8pvmw\" in resourcegroup=\"rg-nightly-4-23-6cpntrp4snvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-6cpntrp4snvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-t8pvmw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-6cpntrp4snvc/cluster-nightly-4-23-t8pvmw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00039c500\u003e: \n failed to create HCP cluster cluster-nightly-4-23-t8pvmw: failed starting cluster creation \"cluster-nightly-4-23-t8pvmw\" in resourcegroup=\"rg-nightly-4-23-6cpntrp4snvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-6cpntrp4snvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-t8pvmw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"048383aba89e8f34e7f8a21014b88ec54df7860e5a0677074f9fe2f6cd08d1bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"0e7c33b837c87670cd35a11641aefa3118943334f501baf884bc7c01141848c7","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"07cd9628a9ccda56f61ac2701b201a766788f87a51e5f01f81c37b36913fcf86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"f7953744450082a3ec5bc94fd5cc7b0b148b3213f668f2afb80a43e6f0953ba9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000806300\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-m52zk49cfnnp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-m52zk49cfnnp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000806300\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-m52zk49cfnnp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-m52zk49cfnnp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"07f6aaec3fac7672daf4b585f66337f950195d8fa05a6b2bb23af2665311ad0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"bc25c842a1235045614d605acc90fea01e39dbaf7d68ff406505237e93f1e16d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ecc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fh8hvw7ft644\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fh8hvw7ft644/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ecc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fh8hvw7ft644\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fh8hvw7ft644/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"12653da51578f335393f75bd8be1c14c1cd9a0c496e1e74a12cefe42d9041e54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"998b87f452d353eeb973a256989175c16ceae9088c3fe30ce72eb5beea463849","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6e100\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-g2znk7bl28vh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g2znk7bl28vh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6e100\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-g2znk7bl28vh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g2znk7bl28vh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"25e70dc21eba7a726695b5e7dca9e393b61d08e6b18e68b4b4a7aa9a630e0560","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"085c6afa73df8e5dcbb8a9e0c59a48a53cb276afb92a752caf002a49c403af63","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2da028d1de19152c6c46d4007133a7a2ed4b9ad7b7bf535f12df8efe6c5cbf1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"252c4afaeebb14c4115f0be666abd104e7adc80f40991bcc6beaa3ff85c91e84","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6a6a0\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qk8wdqf7vwt6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qk8wdqf7vwt6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6a6a0\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qk8wdqf7vwt6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qk8wdqf7vwt6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2e626bd2823c656cd65433a74d8b98d9eea69137b04e48f09664b260fd63c8b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4ba5ad9b16c83f433ab867cff2637f8bd36aa451460f9d15a478f5eb1eda9a65","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e89ce0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e89ce0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"35e4191f9c9df3840d2788badf73c09ac087584d4d9751ebf4f7c7a0bc8b07b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4c2e2bc46dbcb65a27b428b8f5c2170eb0f11ba205842dd52e99ae5bde4b31fc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-scjr8k\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108e260\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-scjr8k: failed starting cluster creation \"cp-ystream-upgrade-4-23-scjr8k\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-pp97jbcxvkl7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-pp97jbcxvkl7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-scjr8k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-scjr8k\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108e260\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-scjr8k: failed starting cluster creation \"cp-ystream-upgrade-4-23-scjr8k\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-pp97jbcxvkl7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-pp97jbcxvkl7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-scjr8k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"36c0921b2a977d8c05ff6fe4383986f223d31a1f4d62a3d5263f885d2b171a42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"677da6ac04d265e57cafd47671de87d4257ef7d06221d84329ebe1ab461fca66","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-q7f6s4\" at install version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d122a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-q7f6s4: failed starting cluster creation \"cp-ystream-upgrade-4-22-q7f6s4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-8bwnk6dktglb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-8bwnk6dktglb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-q7f6s4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-q7f6s4\" at install version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d122a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-q7f6s4: failed starting cluster creation \"cp-ystream-upgrade-4-22-q7f6s4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-8bwnk6dktglb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-8bwnk6dktglb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-q7f6s4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"37fcec25781f3bf8f929f93ecc1da277a11d770f35a9e7bf6d462b996aaeade0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"e1a746b36f9a4f74e169ab5b044f8ffbb4e94a6ac47cce6553d8ffee67ae68fe","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-5rvr88pqswlm/cluster-nightly-5-0-8rxs7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000722280\u003e: \n failed to create HCP cluster cluster-nightly-5-0-8rxs7j: failed starting cluster creation \"cluster-nightly-5-0-8rxs7j\" in resourcegroup=\"rg-nightly-5-0-5rvr88pqswlm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5rvr88pqswlm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-8rxs7j\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-5rvr88pqswlm/cluster-nightly-5-0-8rxs7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000722280\u003e: \n failed to create HCP cluster cluster-nightly-5-0-8rxs7j: failed starting cluster creation \"cluster-nightly-5-0-8rxs7j\" in resourcegroup=\"rg-nightly-5-0-5rvr88pqswlm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5rvr88pqswlm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-8rxs7j\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b2802361a4d1f0550a2f35237ca040bde7897035b1eb8590c35734efdf96198","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"562b8f83f84dba1886b3065181ffee89aab2e6673b7333d41f76aef2eaad1f4e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e20e0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jksvh7q9n6qs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jksvh7q9n6qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e20e0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jksvh7q9n6qs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jksvh7q9n6qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b790d9804ccd86877b520b5dcd3a817fd6ce1ec9a87199871ab3e7cfcb23e52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4f0f437ca045765d2d2b24079491f21e1fd5a85db0b44958c8034b2e66199908","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-6jldpkvl8srb/cluster-nightly-4-21-lcl4f8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001500100\u003e: \n failed to create HCP cluster cluster-nightly-4-21-lcl4f8: failed starting cluster creation \"cluster-nightly-4-21-lcl4f8\" in resourcegroup=\"rg-nightly-4-21-6jldpkvl8srb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-6jldpkvl8srb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-lcl4f8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-6jldpkvl8srb/cluster-nightly-4-21-lcl4f8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001500100\u003e: \n failed to create HCP cluster cluster-nightly-4-21-lcl4f8: failed starting cluster creation \"cluster-nightly-4-21-lcl4f8\" in resourcegroup=\"rg-nightly-4-21-6jldpkvl8srb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-6jldpkvl8srb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-lcl4f8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3c02e3ac9e04dd95492a9f99263378444c794474a16d431e812edc8ada01ffee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"8b5bf6763af34e2d91fc69a112089a080b1da4da152d93af2ebb7174e191d48d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000332240\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-zsvgzkhfdgg2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-zsvgzkhfdgg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000332240\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-zsvgzkhfdgg2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-zsvgzkhfdgg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3c7e04bd3ca5e9d06f4447688470b4fa9bae7e574949f4f88b1835ee68128696","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"f6f1275a16afdec341158ade8fbc4ce79c93972342a64ac61c74f29d86d94b33","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-dtsnl8rgrzrg/cluster-nightly-5-1-47qlh5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c05c0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-47qlh5: failed starting cluster creation \"cluster-nightly-5-1-47qlh5\" in resourcegroup=\"rg-nightly-5-1-dtsnl8rgrzrg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-dtsnl8rgrzrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-47qlh5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-dtsnl8rgrzrg/cluster-nightly-5-1-47qlh5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c05c0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-47qlh5: failed starting cluster creation \"cluster-nightly-5-1-47qlh5\" in resourcegroup=\"rg-nightly-5-1-dtsnl8rgrzrg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-dtsnl8rgrzrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-47qlh5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3e55788f3c5e2e335b5de666590ecce5efbb82bbaf84cc45dd14340cf46b65aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"ffc3ee48d637c5b8ce881fb48a931d89a1627b9e6bc94e2411754ee778248a76","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018260\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-q87tv2dcjtrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-q87tv2dcjtrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018260\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-q87tv2dcjtrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-q87tv2dcjtrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"43007acd125eef4c4f4ba9641cba172cf69e8eb4a96ed853259d5dc1e61e11c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"a4bd95380032fc5885c8b2fd10eb51f3df30c29355f6b414b5b33fef1056e9a7","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-wvjvqc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78220\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-wvjvqc\" in resourcegroup=\"admin-credential-lifecycle-test-hcvns8c6tffl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hcvns8c6tffl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wvjvqc\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-wvjvqc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78220\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-wvjvqc\" in resourcegroup=\"admin-credential-lifecycle-test-hcvns8c6tffl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hcvns8c6tffl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wvjvqc\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4f2c81f7614c8e46a704603c6521f397f4dee37e7f96ee033e32e8cf28574c18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"2740332bfcda380c3c02e8ee1f792b3568b04678a77023d1486f19de701aeb61","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e80120\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-87cdm8lltpfw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-87cdm8lltpfw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e80120\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-87cdm8lltpfw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-87cdm8lltpfw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"505832473c706a2c1d3ea3adee0a3cf10f8bfe4690f9a93c87c10d3f03b19cac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"db766735f9e579f4552ed8a3c29b312be7add829d329efde1debee620ae19b46","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118a260\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ln4schhnc59t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-ln4schhnc59t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118a260\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ln4schhnc59t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-ln4schhnc59t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5377a4023450d40cfd73b5390fc5095150b043936c4a8523045fc7bfc0a1f744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"e94453651233184d87c4057afb3b9acb271e36b21383edb286efa6f234f89bef","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7e460\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-jh9dxccjnhlt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-jh9dxccjnhlt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7e460\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-jh9dxccjnhlt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-jh9dxccjnhlt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"56c9119fdef15adb2d048329c6bdb2240df8b251638dfb06931cc235ef1e6af3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"f0cc45ea9fca99b68f2c8c623264abfb0139dd3f1bc78462db9705e9cff98ef3","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000660600\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-tm274vd7kh49\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-tm274vd7kh49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000660600\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-tm274vd7kh49\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-tm274vd7kh49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5f472f860168a5f3d057ec28ba010ec748982262ba68d107782c88b8355c308d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4d36c1670f69a8378c623f46ed5e24f3787ec0a5de39295ced2ebfe81f29cefa","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010100e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qzw9gg97gq44\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzw9gg97gq44/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010100e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qzw9gg97gq44\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzw9gg97gq44/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5f8ec86cead6bb3bc2fcc82912f45af16f342453ef8de92fb48cfc68da334889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"b44a4a5abf9df504b29cd303adc2b536d491938b5012cb47d37469bc577abd3d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc940\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-lfp5nqml9vq2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-lfp5nqml9vq2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc940\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-lfp5nqml9vq2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-lfp5nqml9vq2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6120fcce0b49a5d6638919802f47669ec822e2213fbbf696f1b32fa8852fe8b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"c0258f3c8ab984584c9b6314be2ccb30e5ffa2b206f4ab774c595c07d6b813f5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-2zfwx7\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000788740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-2zfwx7: failed starting cluster creation \"cp-ystream-upgrade-4-21-2zfwx7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-chltnnssx9l9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-chltnnssx9l9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-2zfwx7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-2zfwx7\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000788740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-2zfwx7: failed starting cluster creation \"cp-ystream-upgrade-4-21-2zfwx7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-chltnnssx9l9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-chltnnssx9l9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-2zfwx7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6285e3b2857f3afd6679f075cb116e7f0ff9d536ea829b0924b09fc59a992953","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"5e002cf10865e514b0815eed29286f8da9237e5aa8330589afaeef395b289efb","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001284320\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7nfk9kblbq5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7nfk9kblbq5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001284320\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7nfk9kblbq5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7nfk9kblbq5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"67d23b1e341dbfc69fc2ef5ac13bb5889247c6c00116dcc32f7ff27a8ccd027a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"27e6130cf36baa158d6711ebe28c73dffa820785f17bbc686ff8fc7d38fbdb8b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ct45g8\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08400\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ct45g8: failed starting cluster creation \"cp-ystream-upgrade-5-0-ct45g8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8wgrfjrhz827\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-8wgrfjrhz827/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-ct45g8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ct45g8\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08400\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ct45g8: failed starting cluster creation \"cp-ystream-upgrade-5-0-ct45g8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8wgrfjrhz827\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-8wgrfjrhz827/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-ct45g8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6de0b7ba8a4640f6e40a7a761bdf178ce8c362a1e9b87c552dd4ce66a983fd06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"cf3269165eebc8b281476a0026c2b3449c115912f9f0f7f746f7524bfed7b58f","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d18100\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9p8lhkrrwxlr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-9p8lhkrrwxlr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d18100\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9p8lhkrrwxlr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-9p8lhkrrwxlr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7f6b4a6e146237cc8039bf67e25a6690cde3da74a4dd8776e7e1b0d6f4f7ea85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"25dbf965d6f20a1a7cec9776ec1e87245a3e50198ba091a7741329a88e0da378","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-q9dmmpg8slh8/cluster-nightly-4-22-cwmzl7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c102c0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-cwmzl7: failed starting cluster creation \"cluster-nightly-4-22-cwmzl7\" in resourcegroup=\"rg-nightly-4-22-q9dmmpg8slh8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-q9dmmpg8slh8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-cwmzl7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-q9dmmpg8slh8/cluster-nightly-4-22-cwmzl7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c102c0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-cwmzl7: failed starting cluster creation \"cluster-nightly-4-22-cwmzl7\" in resourcegroup=\"rg-nightly-4-22-q9dmmpg8slh8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-q9dmmpg8slh8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-cwmzl7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"820c52cf9f44dab35d1b5920159f294cfc24c3032d6c3ca9802f18351f734da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"d66e9a51e6356e3da95cc4251424afb6ce526cd142562c82ca1f05f8b3157c1f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c5a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-cgng7lgtlrhk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-cgng7lgtlrhk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c5a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-cgng7lgtlrhk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-cgng7lgtlrhk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"889a259b17f3e4ce37eb886666d36d8a33f97bb3017c9a0683e8f930703e6f39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"44de21e589e428af479c9ccfb6a72f778841ee48c0b271fb47903606d3baee24","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000992e20\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-7psrwg2xhdxk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-7psrwg2xhdxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000992e20\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-7psrwg2xhdxk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-7psrwg2xhdxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8c62ce3d04cc6b143e48077b2116edc60f7ad85af542023142389a7a2bc452d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"cb988d58c89113961f03df82043150415e9b6ed049bf1f878c0c0b5c9eb97486","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a06a20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-dpsml89bt4jr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dpsml89bt4jr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a06a20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-dpsml89bt4jr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dpsml89bt4jr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"926eb6198a8189f9d90a779dbf850c42e2e01877e92fcda3dfca1e0e829b3a7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"483e21afcf9632ea6968f51c68da44c7e6f45d9a5418bb6f513b2bec6b1bbce2","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000984180\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-lc76mwwz2m4r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-lc76mwwz2m4r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000984180\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-lc76mwwz2m4r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-lc76mwwz2m4r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9b1a0b99e91a0e8f57800f5bf76884cfc81108765759c9fc2573db7d89e45348","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"ce5b35b01c844ca412a4cc92791058f44a790b6ff65cc19fb352ae11b15d2ae3","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-s8bnbc\" at install version 5.0.0-0.nightly-multi-2026-09-16-151028\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a81e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-s8bnbc: failed starting cluster creation \"cp-ystream-upgrade-5-1-s8bnbc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-gt6bnlnww2lj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-gt6bnlnww2lj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-s8bnbc\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-s8bnbc\" at install version 5.0.0-0.nightly-multi-2026-09-16-151028\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a81e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-s8bnbc: failed starting cluster creation \"cp-ystream-upgrade-5-1-s8bnbc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-gt6bnlnww2lj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-gt6bnlnww2lj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-s8bnbc\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a1a65e603922e7bd490f987bc8ff1ed9627935392c14054a0ab317ad0c02d790","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"265ce4c4fa80432d1672154f22571f629b7a60e7c351781f21d111b2d1880927","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001458160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-462w74wptjnv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-462w74wptjnv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001458160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-462w74wptjnv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-462w74wptjnv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a21c5321007ba8f534c3cda3b27998c74c2101914ec8be8f082ac4b72348bfdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"a9d975f128c27c084e7e5fd23daa85c45a8d628e9193f529805c3f218b289677","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-hh4kqh at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e303e0\u003e: \n failed to create HCP cluster np-skip-minor-hh4kqh: failed starting cluster creation \"np-skip-minor-hh4kqh\" in resourcegroup=\"rg-np-skip-minor-hh4kqh-g8ngpxdpgdxv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-hh4kqh-g8ngpxdpgdxv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-hh4kqh\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-hh4kqh at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e303e0\u003e: \n failed to create HCP cluster np-skip-minor-hh4kqh: failed starting cluster creation \"np-skip-minor-hh4kqh\" in resourcegroup=\"rg-np-skip-minor-hh4kqh-g8ngpxdpgdxv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-hh4kqh-g8ngpxdpgdxv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-hh4kqh\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aaee0fa9fcd431d85846a8340c0164b202624864199d9a4680797450e7787be7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"340421cbdb7899927b7ac5fc7d0aa57d005e6eb5d784359ad31a36aed34461d6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e86cc0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-v5tthrvp4l2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e86cc0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-v5tthrvp4l2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b0edfb21904a722110dc7fa2fc6e955e8b7f9b00e18afb4233f7a59e235e6c1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"f0973137271b8ecc421edadd409b7fe63accbff0882cef646035b79a0aa8c22e","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011fc200\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-cm746plzmjgp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-cm746plzmjgp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011fc200\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-cm746plzmjgp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-cm746plzmjgp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b208f30245ffd27f4465ee5a583f68bb6e52eedb1ed114a06ece61ff0ca1cd72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"58f07f406d642a91c9bddbe8fdd7ede440f08a2dd03a6f5a5c819b498466cf11","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e0c0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lszv6zzms7rw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lszv6zzms7rw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e0c0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lszv6zzms7rw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lszv6zzms7rw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bc051456b4bc2739e4a61f2644b2532fc787af21c3dad32e14ba1a3a043d4115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"71e62b511a0fe354af7045a804df32da837a8d8a1285d20120469dc2b9b474a1","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010100e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-qwjk9ftzxk8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-qwjk9ftzxk8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010100e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-qwjk9ftzxk8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-qwjk9ftzxk8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"be541cd200ddd024fa3ae0f9b965d5f5e2906702669a4a85871fc0f27b16f152","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"10f37d08fe2d36b0048e252aedb2d548e4ca8db071149c417d5ce17b13fa36d9","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-q825bz with version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a251e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-q825bz: failed starting cluster creation \"np-version-upgrade-cluster-q825bz\" in resourcegroup=\"rg-np-version-upgrade-q825bz-x27zct4fpv9n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-q825bz-x27zct4fpv9n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-q825bz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-q825bz with version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a251e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-q825bz: failed starting cluster creation \"np-version-upgrade-cluster-q825bz\" in resourcegroup=\"rg-np-version-upgrade-q825bz-x27zct4fpv9n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-q825bz-x27zct4fpv9n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-q825bz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c9e9f383cffa2eeb84dfa964a5ece9504c18dda1511de99097692432c6312c1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"c3410d264cf0132fe4f17940cdf2ec821ec784b23b0faff95c971f7658965387","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000804820\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-hfrqt4znr8ww\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-hfrqt4znr8ww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000804820\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-hfrqt4znr8ww\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-hfrqt4znr8ww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d672ec37399e623567430280538d275a91c4d26ab870f8fccc07c8928fbd97b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4455b438d122ea70538db65aa4dcf60523e82a18567f93d3872e7891ddeb2d9f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-stz6qmm9c9pm/cluster-nightly-4-20-jhmr4f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c160\u003e: \n failed to create HCP cluster cluster-nightly-4-20-jhmr4f: failed starting cluster creation \"cluster-nightly-4-20-jhmr4f\" in resourcegroup=\"rg-nightly-4-20-stz6qmm9c9pm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-stz6qmm9c9pm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-jhmr4f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-stz6qmm9c9pm/cluster-nightly-4-20-jhmr4f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c160\u003e: \n failed to create HCP cluster cluster-nightly-4-20-jhmr4f: failed starting cluster creation \"cluster-nightly-4-20-jhmr4f\" in resourcegroup=\"rg-nightly-4-20-stz6qmm9c9pm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-stz6qmm9c9pm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-jhmr4f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d86e8627f71193a2dbd3083852ab1c9da7f2aab28b47f69479e8a446fe17c963","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"0f82bd0c9700cf4a07c13d1f48415f495ffac7b49c2a7ae3841ea9c8faf79ca5","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d9536ecebb563f79dd5fa1f8145ef4f45494e0734f1964c541e5cfc2bb9c49a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"c68022f2f40c275eb8c23a59749c7f7e4af0d44ada9136f9192d94b24e86254b","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fe300\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-68hbxkjb8pb5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-68hbxkjb8pb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fe300\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-68hbxkjb8pb5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-68hbxkjb8pb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d9976cdfe7de8916376eb1a5a815acfad947cc17d0d302ded1fca77f7ec42ee3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"3f18ddd10272127c408a7d9bde4eddaaa27e6f085f80cf7d94ab9be21a380754","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a29c0\u003e: \n failed to create HCP cluster np-dg-minor-88hvpm: failed starting cluster creation \"np-dg-minor-88hvpm\" in resourcegroup=\"rg-np-dg-minor-88hvpm-hhs8h4j7xp4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-88hvpm-hhs8h4j7xp4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-88hvpm\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a29c0\u003e: \n failed to create HCP cluster np-dg-minor-88hvpm: failed starting cluster creation \"np-dg-minor-88hvpm\" in resourcegroup=\"rg-np-dg-minor-88hvpm-hhs8h4j7xp4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-88hvpm-hhs8h4j7xp4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-88hvpm\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"de54513f084dd154621123a0cb677bebb415e92e4cc27bf434bea340964e19c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"71d45e9348cce618a98ba3a9cedd12637edd7a5d0db05784f92b3cf08108bd98","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb4180\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-qbtxlbn5dcbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-qbtxlbn5dcbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb4180\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-qbtxlbn5dcbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-qbtxlbn5dcbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e7095052370a7ab5d0139ab379052e8504cb0e03194859aaba4069a3c82e9516","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"b89878daa0c4d5d3cb3d93a46690c3d9ddf063236bf34cd2f69bcdf3bb5f245a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0100\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-5sqvdl7975w5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-5sqvdl7975w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0100\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-5sqvdl7975w5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-5sqvdl7975w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e82cded1e6059a42d27b0c882a2c589d9584eb7c6ec349124e24b616efd8cc34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"42f7a4e6cab05c380f4d5d815367e49f29a7e4b22dcb36d855a4a05388b7e8cf","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a80140\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hndxmlgn7dsk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-hndxmlgn7dsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a80140\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hndxmlgn7dsk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-hndxmlgn7dsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e9214061eb2c610cad99f734d2fb20000261e718b868cbedb6817da5eddb41ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"6adb067c216551bfe29f1e3b77d6113e9d3d513c52d7c6b6659f174eed5dbd71","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00156e300\u003e: \n failed to create HCP cluster ea-listvrnw2h: failed starting cluster creation \"ea-listvrnw2h\" in resourcegroup=\"ea-list-gqc5zfmw9m5t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-gqc5zfmw9m5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listvrnw2h\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00156e300\u003e: \n failed to create HCP cluster ea-listvrnw2h: failed starting cluster creation \"ea-listvrnw2h\" in resourcegroup=\"ea-list-gqc5zfmw9m5t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-gqc5zfmw9m5t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listvrnw2h\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f84c47f0bdb39c4d772bc628a7b5c7069691d90fbd5c524e2b38f91651bad3fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"1dd5e0dc7e15d356bc9a171360fd68f57e762e2e53e6cea924436fb68b6fb102","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad80e0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-x4gcgll7nktk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-x4gcgll7nktk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad80e0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-x4gcgll7nktk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-x4gcgll7nktk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fc47a3a302fa4fe7906a212d7205491c5487e3181bd19eac993a4758c0d8edfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"97623f5d3f0d0fd367349a611b76873782d9adb09766bc94fe24252159b1769d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000333e80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mkvhgjv66g9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-mkvhgjv66g9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000333e80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mkvhgjv66g9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-mkvhgjv66g9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ff3666e90a287132950b72e77d00d56ab3f7412aa79f661e209d7e20814142ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"1153df3f24dc8bcdeecfdd99c8141522fe1f0d9dd14f566bee678a30941ac49f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004badc0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-78mvmfhw2cmf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-78mvmfhw2cmf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004badc0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-78mvmfhw2cmf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-78mvmfhw2cmf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:00:17Z","started_at":"2026-09-18T02:00:17Z","completed_at":"2026-09-18T03:56:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:56.226117142Z","timing_metadata_checked_at":"2026-09-18T15:42:56.226117142Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:56.229309553Z","region_metadata_checked_at":"2026-09-18T15:42:56.229309553Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5f91417f322bd99b242e4b524a260be5de89edae53ca2f3a61041a3c276b8a84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"ea0f58ce05963824cd5712a4a308420855630745ad3e354951e106ba980a6661","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded"}},{"row_id":"9cae8edd1d29a8636dad08d02205af168fd2d8aa92320a00c8fca46c397abbfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"16ebc00197f5d484a478287dbcf774e95611e6748652e8b6528ea5b7d11de1a1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}