{"meta":{"date":"2026-09-18","timezone":"UTC","generated_at":"2026-10-31T08:32:18Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":84,"failed_runs":59,"runs_with_occurrences":59,"runs_with_failure_pattern_matches":49,"runs_with_unmatched_occurrences":10,"runs_without_occurrence_rows":25,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7017,"pr_state":"closed","pr_sha":"55788c289a67f3ac476a0ae85bf3de8260953664","final_merged_sha":"f177240b9d6b4d251ea43939d1ec48ab5b62d5f0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T23:03:02Z","started_at":"2026-09-18T23:03:02Z","completed_at":"2026-09-19T00:54:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.764675578Z","timing_metadata_checked_at":"2026-09-19T03:30:43.764675578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.768173891Z","region_metadata_checked_at":"2026-09-19T03:30:43.768173891Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"dd6353aebccaeca7eaadd48d97bcda89c5d2979821c82bceb1cfd6aa862e8e1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","occurred_at":"2026-09-18T23:03:02Z","failed_at":"alert","signature_id":"86c45d947e908dd24da56e029e4fb3fb50ede5048c717aadd07439573c0a5e5a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:47:51Z\n Ended: 2026-09-19T00:19:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8441088-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8441088-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7012,"pr_state":"closed","pr_sha":"2069fe0ee5d2249364832dbb2c3166d5a85d96e9","final_merged_sha":"2069fe0ee5d2249364832dbb2c3166d5a85d96e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T22:44:47Z","started_at":"2026-09-18T22:44:47Z","completed_at":"2026-09-19T00:36:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.136011775Z","timing_metadata_checked_at":"2026-09-19T03:30:43.136011775Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.139157486Z","region_metadata_checked_at":"2026-09-19T03:30:43.139157486Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"139b92f24a913d47aa240a4e218bdc9e4dbfef5ab826c9b8d524e6c859656832","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"a46151e80a37451e13b91d43941493c16950cf32997bf773ab888a8c5db06db6","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"579e01f42e94dee6a803ea631eada6ef71b7974e00977d9678afb887f3b1f56d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"7dead5f17d20c600ef9a15a67c9915db561b768a43db7c68ada9d07b7b90dfc6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:17:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:18:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"99ae21d96735666007952a512d8240b0d5951d0f234954f6074bb56dd7ba272f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"e2e","signature_id":"2c332da578489a812e0144aacf458767c886ed8769b126097a13ac0573609822","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d503582cd5359ec3a787862a0d3ae2209b75489992f83d22ae2e8e10e463450b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"61a9d86a254d9ba501b68a3f3cb4e321b30d8b314bb25ec5b72e60af5be20fd0","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:30Z\n Ended: 2026-09-19T00:23:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"e18a02cc81ba5f8eb087e5af03c24ecfeee3031e86b210b833a22ae3272e3914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","failed_at":"alert","signature_id":"b0cb1249b73e4e17694cee1167aadd3f0fe5951d24186dea9efdb1984ff09dea","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:52:29Z\n Ended: 2026-09-19T00:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:53:28Z\n Ended: 2026-09-19T00:17:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101068742900322304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T21:59:58Z","started_at":"2026-09-18T21:59:58Z","completed_at":"2026-09-18T22:08:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.929506818Z","timing_metadata_checked_at":"2026-09-19T03:30:42.929506818Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-19T03:30:42.933952334Z","region_metadata_checked_at":"2026-09-19T04:34:06.858678691Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"414fd554e1a3395cbd82b855d7b7d647f94ff634a4ae2d270b726ebc1dfd13f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101068742900322304","occurred_at":"2026-09-18T21:59:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101065688771727360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"e7d302ddca8bb69f3f50165f6260d152b11bef67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:47:50Z","started_at":"2026-09-18T21:47:50Z","completed_at":"2026-09-18T21:56:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:39.889619581Z","timing_metadata_checked_at":"2026-09-19T03:30:39.889619581Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-19T03:30:39.894277798Z","region_metadata_checked_at":"2026-09-19T04:34:06.766254951Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f60224742cddf98497cedc0676c1042f2d25ee70b80e43834ed07fd17d181fc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101065688771727360","occurred_at":"2026-09-18T21:47:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"eae56fbadc2e1af941a3cecdb38bd7071e123b2c","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:36:30Z","started_at":"2026-09-18T21:36:30Z","completed_at":"2026-09-18T21:57:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:38.738880258Z","timing_metadata_checked_at":"2026-09-19T03:30:38.738880258Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:38.743353974Z","region_metadata_checked_at":"2026-09-19T03:30:38.743353974Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c3796091f6a6408c84986b62fb5fbfadcdf976575bddfe72195f2c827435c058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","occurred_at":"2026-09-18T21:36:30Z","failed_at":"provision","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-18T21:50:35.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T21:51:25.579Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"name\\\": \\\"04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'aad2e336-6c43-473c-997c-4a60a482583a'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4937088/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26091821etad3\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: b8850252-9aac-448f-9581-e29b2b8a10a8.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"open","pr_sha":"96c7afd52028f383231ff4e814218fb4e6b898b6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T21:22:20Z","started_at":"2026-09-18T21:22:20Z","completed_at":"2026-09-18T22:06:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.029064802Z","timing_metadata_checked_at":"2026-09-19T03:30:33.029064802Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.032546015Z","region_metadata_checked_at":"2026-09-19T03:30:33.032546015Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fd8055da033db00b4f97ef1c80a305e861fad1ca6a1da0262b2e7f9c777833d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","occurred_at":"2026-09-18T21:22:20Z","failed_at":"provision","signature_id":"b8972dd0ff9a3ca0c20207306d152fa4afd8e4eb6a37b8d1c512cde87ef5ae3e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-09-18T21:55:47.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-6 clientId:9c78a811-9c85-4eb0-9c84-d0843866c888 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj1086464.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj1086464.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Enabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f databaseName:arohcpci01-rp-j1086464 databaseUrl:https://arohcpci01-rp-j1086464.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-vh2hkbkg/pipeline@sha256:ec4f5311c601606f0bdcb4316ea9dff923b912c8e13a93addd5ac2c23dd9e86c replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:475m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-12 clientId:50d052a6-7aec-48b9-b43e-b8692a8f0ead clientIdSecretName: principalId:a76148d4-cb11-4b93-9363-54ba8239b0b1 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-vh2hkbkg/pipeline:pull workloadIdentityClientId:09708096-ca85-4992-8eb5-667a9b9d3fce workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:48.698Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:49.102Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"closed","pr_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","final_merged_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T21:00:47Z","started_at":"2026-09-18T21:00:47Z","completed_at":"2026-09-18T23:02:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.49756873Z","timing_metadata_checked_at":"2026-09-19T03:30:33.49756873Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.503477052Z","region_metadata_checked_at":"2026-09-19T03:30:33.503477052Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"25dae7a2a8aa574e5cc36a74517384b48ea9b23b21c576e82f1ba3cd82d07894","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","failed_at":"e2e","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"949106af16941bfb6470bdea16b80d0f435a326819d0ac895aa85186d1332c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","failed_at":"e2e","signature_id":"6566438295d27151c619c08c959579eb7709fb58c49e3c34c87dd1ec1d1e808b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053931038707712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T21:00:47Z","started_at":"2026-09-18T21:00:47Z","completed_at":"2026-09-18T23:12:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.319667784Z","timing_metadata_checked_at":"2026-09-19T03:30:42.319667784Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.323153197Z","region_metadata_checked_at":"2026-09-19T03:30:42.323153197Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7018,"pr_state":"open","pr_sha":"5c4e7b27f7d76d1aac25b9e8c9d004c3a38882f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T20:43:02Z","started_at":"2026-09-18T20:43:02Z","completed_at":"2026-09-18T22:38:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.902194082Z","timing_metadata_checked_at":"2026-09-19T03:30:43.902194082Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.905842595Z","region_metadata_checked_at":"2026-09-19T03:30:43.905842595Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0b55226030c8b821323f6ec09b006063aa5cd9671cb923b54989f1746921039e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","occurred_at":"2026-09-18T20:43:02Z","failed_at":"alert","signature_id":"900336073fa1807e3fa5eb270059d6e459df8256995e512e7d9ae978fff82f8c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T22:08:21Z\n Ended: 2026-09-18T22:18:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4790528-svc\", component=\"frontend\", method=\"delete\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4790528-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:52:27Z","started_at":"2026-09-18T19:52:27Z","completed_at":"2026-09-18T21:47:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:38.280963169Z","timing_metadata_checked_at":"2026-09-19T03:30:38.280963169Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:38.284919484Z","region_metadata_checked_at":"2026-09-19T03:30:38.284919484Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"07cd0904674d1540cc7d921fe4ddc5469c54ab0f3616558dfbe48b81bbce5f6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","occurred_at":"2026-09-18T19:52:27Z","failed_at":"alert","signature_id":"f8ce7b6dbb00b186dd3690b8bd0cde21b1999ed29a53641a6363be8c9bc8013e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T21:03:41Z\n Ended: 2026-09-18T21:09:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6750464-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6750464-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"e7d302ddca8bb69f3f50165f6260d152b11bef67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T19:39:52Z","started_at":"2026-09-18T19:39:52Z","completed_at":"2026-09-18T21:45:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:39.761505011Z","timing_metadata_checked_at":"2026-09-19T03:30:39.761505011Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:39.765341625Z","region_metadata_checked_at":"2026-09-19T03:30:39.765341625Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"155e4bac73019064e828e68620a1f1e6bb58c9e92c4ecfcb44479894745088e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","occurred_at":"2026-09-18T19:39:52Z","failed_at":"e2e","signature_id":"50e9ecb754e15646a314b5b99f973f998a45c92fb4fac3293bbdba2c5f6a2e7e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101024271722614784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T19:03:16Z","started_at":"2026-09-18T19:03:16Z","completed_at":"2026-09-18T21:03:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:40.405968472Z","timing_metadata_checked_at":"2026-09-19T03:30:40.405968472Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:40.409069784Z","region_metadata_checked_at":"2026-09-19T03:30:40.409069784Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023786441641984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T19:01:20Z","started_at":"2026-09-18T19:01:20Z","completed_at":"2026-09-18T21:18:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:34.081150382Z","timing_metadata_checked_at":"2026-09-19T03:30:34.081150382Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:34.085552198Z","region_metadata_checked_at":"2026-09-19T03:30:34.085552198Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:01:12Z","started_at":"2026-09-18T19:01:12Z","completed_at":"2026-09-18T20:12:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.149154859Z","timing_metadata_checked_at":"2026-09-19T03:30:42.149154859Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.152896373Z","region_metadata_checked_at":"2026-09-19T03:30:42.152896373Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1a1376dd0fb95d0eb56f649393a3cb9cf0da57657aae154c8d3840d96661bee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","occurred_at":"2026-09-18T19:01:12Z","failed_at":"provision","signature_id":"30e287a059c7694cef227767592b23353845f4281b98b8c7ff98febc06270f17","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-18T20:01:01.932Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 values=\"map[enabled:true image:map[digest:sha256:5177425b194b5d289664c62bedbdc95a42605bbfcde48fc7ddbb31bee0194b90 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.574Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 namespace=monitoring\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 dryRun=false\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.827Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T19:01:10Z","started_at":"2026-09-18T19:01:10Z","completed_at":"2026-09-18T21:16:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.669254465Z","timing_metadata_checked_at":"2026-09-19T03:30:42.669254465Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.672341176Z","region_metadata_checked_at":"2026-09-19T03:30:42.672341176Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"51d08b7a9ce6eb284dd77abd1ad96549aa4cd646cf58c08579dd431de0cb4b96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","failed_at":"alert","signature_id":"4beb77cb7f662a3ba0c82df6677c24959b741d146512bce1d7d72b33bd6e9a0e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T20:11:59Z\n Ended: 2026-09-18T20:26:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6109056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s\", pod=\"router-8444b6f56d-w6qnw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s/router-8444b6f56d-w6qnw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"59b46d6e51f79fb2f2cef722ec2d3bcef1e24821ac34f57d76534720f970aae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","failed_at":"e2e","signature_id":"c0b449ac08413f5f7083190089ef0d839099fa2699d64d1c9f2f8d26abbef529","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"closed","pr_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","final_merged_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T18:24:36Z","started_at":"2026-09-18T18:24:36Z","completed_at":"2026-09-18T20:20:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.31576216Z","timing_metadata_checked_at":"2026-09-19T03:30:33.31576216Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.327947105Z","region_metadata_checked_at":"2026-09-19T03:30:33.327947105Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c325b1238b2a4925b518747db89630c9a558c69c2b0b732e3b1da306fbbc469c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","occurred_at":"2026-09-18T18:24:36Z","failed_at":"e2e","signature_id":"96d0afdee41d99182089b8922944bd77cbf90a6f0b4ca2fe3e238fbfdaae440e","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7017,"pr_state":"closed","pr_sha":"55788c289a67f3ac476a0ae85bf3de8260953664","final_merged_sha":"f177240b9d6b4d251ea43939d1ec48ab5b62d5f0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T18:23:12Z","started_at":"2026-09-18T18:23:12Z","completed_at":"2026-09-18T18:56:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.598267568Z","timing_metadata_checked_at":"2026-09-19T03:30:43.598267568Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.603919489Z","region_metadata_checked_at":"2026-09-19T03:30:43.603919489Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c9cc4d15c1da0be79b46d0992c58977c3ef41fb98246c274c3cbea10aa1f2fd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","occurred_at":"2026-09-18T18:23:12Z","failed_at":"provision","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","test_suite":"step graph","failure_text":"time=2026-09-18T18:49:43.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n Kind: ARM\\n Template: templates/svc-infra-lookup.bicep\\n Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-18T18:49:44.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-18T18:49:45.666Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6366720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd\ntime=2026-09-18T18:50:16.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6366720-svc/providers/Microsoft.Resources/deployments/299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd/operationStatuses/08584118495004368821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7010,"pr_state":"closed","pr_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","final_merged_sha":"b9f8fb127a8a256707e3aaeb2c00bb8265cf06de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:36:50Z","started_at":"2026-09-18T16:36:50Z","completed_at":"2026-09-18T18:38:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:42.48515329Z","timing_metadata_checked_at":"2026-09-19T03:30:42.48515329Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:42.488768903Z","region_metadata_checked_at":"2026-09-19T03:30:42.488768903Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2d03db804d08a5f94e0f525a3a7e2944810909b2021943645284f702064c910b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"adf91e76f1e8a11039895567a05fb0e5c3f1d56a9565f8cf0e71b7f1f6e5d0ce","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:37:21Z\n Ended: 2026-09-18T18:05:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9556480-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress\", pod=\"router-8495fb9c6b-n57p2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress/router-8495fb9c6b-n57p2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"499a269e139f43071276772a1c9a391a3bcc57585f049d433ae1cd506717be87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"e2e","signature_id":"99f885a8a863b5de08d9591137eee29d3c46e0095bea1b9e8046d833c161bfec","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"655707720d06a4babc91b86cb1ae99605111f9dd203a1eb99acc6c36f5dadcca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"1522dde0b9bb5d85e613b7efec8820e9ced25e3c11ff2b7a1c60a5642910649a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:56:41Z\n Ended: 2026-09-18T18:02:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9556480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"90fa4d791a743565dcbc1e61511ee86c5e89d4a029763e85151f644a46563bcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","failed_at":"alert","signature_id":"f6e1d34656f1ce37365020fae4967ac3b4ae64788fb876cb79e6b050b731b2fe","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T18:04:41Z\n Ended: 2026-09-18T18:24:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9556480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100986377096138752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6805,"pr_state":"closed","pr_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","final_merged_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T16:17:54Z","started_at":"2026-09-18T16:17:54Z","completed_at":"2026-09-18T18:36:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:34.879501226Z","timing_metadata_checked_at":"2026-09-19T03:30:34.879501226Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:34.88329454Z","region_metadata_checked_at":"2026-09-19T03:30:34.88329454Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:10:25Z","started_at":"2026-09-18T16:10:25Z","completed_at":"2026-09-18T18:25:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:33.93947626Z","timing_metadata_checked_at":"2026-09-19T03:30:33.93947626Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:33.947493889Z","region_metadata_checked_at":"2026-09-19T03:30:33.947493889Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6fd9d882d77dd6c95fcad26bf18ad303ee02e15867f25ce324525bcdeb08b73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","occurred_at":"2026-09-18T16:10:25Z","failed_at":"alert","signature_id":"f7df40a156b9e56e6a5ddf150c7ac6e35049cd21802df2d33f93003f72173db8","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:42:43Z\n Ended: 2026-09-18T17:48:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3570304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3570304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T16:01:55Z","started_at":"2026-09-18T16:01:55Z","completed_at":"2026-09-18T18:45:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:41.705962035Z","timing_metadata_checked_at":"2026-09-19T03:30:41.705962035Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:41.710943154Z","region_metadata_checked_at":"2026-09-19T03:30:41.710943154Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b78d7fc419a09e5f4bd131af5dfb921f08e79bbc69566ea089cccfc2d63a07c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","occurred_at":"2026-09-18T16:01:55Z","failed_at":"alert","signature_id":"db6a9f6935fcb92fd2604ae71dc7832d2001b5cc7b8edb8be0b3faebabe391c0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:59:03Z\n Ended: 2026-09-18T18:05:04Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2544640-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2544640-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7014/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100985423357546496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7014,"pr_state":"closed","pr_sha":"1411888568eb8c72f1a939fe6076c3f61faaa630","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:57:16Z","started_at":"2026-09-18T15:57:16Z","completed_at":"2026-09-18T18:33:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:43.463439674Z","timing_metadata_checked_at":"2026-09-19T03:30:43.463439674Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:43.466385185Z","region_metadata_checked_at":"2026-09-19T03:30:43.466385185Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"fef1b259e6981712a9c624e9eab0c55d83f17aeb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T15:17:47Z","started_at":"2026-09-18T15:17:47Z","completed_at":"2026-09-18T18:36:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:35.860472443Z","timing_metadata_checked_at":"2026-09-19T03:30:35.860472443Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:35.864219257Z","region_metadata_checked_at":"2026-09-19T03:30:35.864219257Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1948be335692ad9a124313808e2157ccdb05236054f536818bd71b0afb7722f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"9e65f4af22bf7ec9953560fed8bb49590fa82fc63d7f9f6d24d707a1a503f81c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2657095b66e4cc164c41d453df5451025086cf7e54a24d404a46420fefe7d7e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"alert","signature_id":"035123031fa1d20ace889ddb3bd473206d7c64b2e0e69a0775e5e52e6c40e898","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:03:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"880ec577c2d949332cde1d90917ab779ddc5c423ce63817f14fb1fbbaae01d67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"alert","signature_id":"b2d9d84ce6e1c74ea14279f086b91ae4fc06d24e40b82a7ad2aa54e78ffa7d4b","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:04:42Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease governance-policy-framework in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease config-policy-controller in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"8c9023598adfb161da69a9a1b49b6b4e86180078bc9c1dcbe9aac1655663ab5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"a0f110873c9fc7db9f376a400bf0d8b83aa3eddcd644b62e99f4963c1f15aa39","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c46e909b0e6f4e8a1c9afd46392acc4d6cfdb0c1397ae5723a896270d73c4af7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"7e8e740680a98bd62be8f0f23d65fc40251d4d5f7742370deded696800b1b588","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5202a8e261350ac547743f88e12674d7803f9696bbba338e24743bc31c9862c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","failed_at":"e2e","signature_id":"52f64f30d917c25349ded7085a1164f0e2615c20f54359a71e907e0ab61c033a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100982457594548224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6858,"pr_state":"open","pr_sha":"150b9bf5d82a4bcc1f959e465ffe6769e739370b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:08:59Z","started_at":"2026-09-18T15:08:59Z","completed_at":"2026-09-18T18:15:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:35.595768767Z","timing_metadata_checked_at":"2026-09-19T03:30:35.595768767Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:35.599651282Z","region_metadata_checked_at":"2026-09-19T03:30:35.599651282Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100965684178587648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"dde72f49803d49335aafd35b01b79b6a92ac9d27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T15:08:34Z","started_at":"2026-09-18T15:08:34Z","completed_at":"2026-09-18T17:17:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:36.532213921Z","timing_metadata_checked_at":"2026-09-19T03:30:36.532213921Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:36.535544433Z","region_metadata_checked_at":"2026-09-19T03:30:36.535544433Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"88cc31ebdf51e981ac966aab5fe4333c0fee21a7","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T15:08:20Z","started_at":"2026-09-18T15:08:20Z","completed_at":"2026-09-18T15:49:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:50:44.615833908Z","timing_metadata_checked_at":"2026-09-18T15:50:44.615833908Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:50:44.618971719Z","region_metadata_checked_at":"2026-09-18T15:50:44.618971719Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"24e43b78ca91d5bc59c5953524fe878e8eb8d41d5a26884bb53457353386e07f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","occurred_at":"2026-09-18T15:08:20Z","failed_at":"provision","signature_id":"e4c3fb1587b09488a2265155d7a6787ca94e35b729bce621d58ebb8765ea7ef1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-18T15:40:00.669Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T15:42:51.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"name\\\": \\\"326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'e78fa208-8c43-45b6-a80c-57ad93c6a044'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4742016/providers/Microsoft.Resources/deployments/ka7tdccaiavlrtibmy3y5s7tfhcs6gpzxc6p4xe5pchjy-26091815bptz8\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: d9b78f81-a4e0-4b15-9e4d-75e6f5c342e3.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100972082392207360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T15:02:13Z","started_at":"2026-09-18T15:02:13Z","completed_at":"2026-09-18T17:27:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:37.673985331Z","timing_metadata_checked_at":"2026-09-19T03:30:37.673985331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:37.678980149Z","region_metadata_checked_at":"2026-09-19T03:30:37.678980149Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100972655304773632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T15:01:55Z","started_at":"2026-09-18T15:01:55Z","completed_at":"2026-09-18T17:36:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:37.325413146Z","timing_metadata_checked_at":"2026-09-19T03:30:37.325413146Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:37.32923696Z","region_metadata_checked_at":"2026-09-19T03:30:37.32923696Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T13:17:26Z","started_at":"2026-09-18T13:17:26Z","completed_at":"2026-09-18T15:21:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.925955013Z","timing_metadata_checked_at":"2026-09-18T15:41:21.925955013Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.929088124Z","region_metadata_checked_at":"2026-09-18T15:41:21.929088124Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"87ca00630dc338a1e8f9db92f36c7acce78945e58c03404050bd9733bacd7b92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","occurred_at":"2026-09-18T13:17:26Z","failed_at":"alert","signature_id":"d6f983561c2168d8dc1fb0bb24a4ce58dae5f6ea67b850648a086ef0cc8ced1e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T14:32:08Z\n Ended: 2026-09-18T14:39:08Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1716224-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1716224-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"91261d5b93a8d4039291003288ddae318a2a8506","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T12:56:34Z","started_at":"2026-09-18T12:56:34Z","completed_at":"2026-09-18T15:14:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.599107407Z","timing_metadata_checked_at":"2026-09-18T15:41:20.599107407Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.603177722Z","region_metadata_checked_at":"2026-09-18T15:41:20.603177722Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7112400e1b8f88fe771ccb3807b8cc948c099d4421e9d140be9b9294fad6bf07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","occurred_at":"2026-09-18T12:56:34Z","failed_at":"e2e","signature_id":"946ebd2cdc29e94011829e7ef21165a52d1801fd362344870c8d6a3e7c5269c4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100926281322336256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T12:33:53Z","started_at":"2026-09-18T12:33:53Z","completed_at":"2026-09-18T14:30:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.215063671Z","timing_metadata_checked_at":"2026-09-18T15:40:40.215063671Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.218157282Z","region_metadata_checked_at":"2026-09-18T15:40:40.218157282Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T11:57:07Z","started_at":"2026-09-18T11:57:07Z","completed_at":"2026-09-18T14:00:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.612275794Z","timing_metadata_checked_at":"2026-09-18T15:40:45.612275794Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.615271905Z","region_metadata_checked_at":"2026-09-18T15:40:45.615271905Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"99fbae33497987b2ca58c464dd69244aac980463b1109cfdccc4b526732c17d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","occurred_at":"2026-09-18T11:57:07Z","failed_at":"alert","signature_id":"85fef6c172f8238e32065ce9be94996092fd65499dbb48afe109749b4511663e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:52Z\n Ended: 2026-09-18T13:38:52Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6576768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6576768-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T11:54:51Z","started_at":"2026-09-18T11:54:51Z","completed_at":"2026-09-18T14:07:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.271599771Z","timing_metadata_checked_at":"2026-09-18T15:40:44.271599771Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.275509786Z","region_metadata_checked_at":"2026-09-18T15:40:44.275509786Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0b321804d1b8c11acab93194bb72fc88d0f4a78a49c438ee42b7c414ef5d9dde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"991b7eb9b2760771a64ba10515aeb5d1b17b39f0d0d776abbc32c3f1fb85abb3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (2 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:05:51Z\n Ended: 2026-09-18T13:34:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j\", pod=\"router-86c7fcbd7d-pjgqb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j/router-86c7fcbd7d-pjgqb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:07:43Z\n Ended: 2026-09-18T13:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster\", pod=\"router-7c5f44b8f5-2rxm6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster/router-7c5f44b8f5-2rxm6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"28bce12764c4b9c1d361a139c92d92bb565be67867f79583c26a96a6bda7f9aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"77db7532989dc335d44e3fd0e347138b7a4aa37045fd816ecfc3761a0789b616","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:44:43Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:45:42Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"39638348eb75451765766ab378ccb201ddf353daa0248edb4b62dfb6c29fb70d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"803dd3435e0189d03ce01d4d7de3f8e741a5ee356c8001e19b84595bbaa71298","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"409e38279d0775df031e734a590194632f5e380be587192046ffde98a8006e92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"4367b100b20534920c26dfaf7f8f61e0605f9aa71cbb971e026f2f07d8c0892b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"f71f9d0966fc26ad6ff3cd9a2f086d86818c4234b98d4c027fd22b942a6e7ba2","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:16:41Z\n Ended: 2026-09-18T13:44:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:17:41Z\n Ended: 2026-09-18T13:45:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"837b8358c268b512a4274ba3da67b785913fe6d7e617731945b4423b0c563171","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"alert","signature_id":"b30257b5749c88e7acf4c108618d88d0f18563968e68f249086f8fef88556ee6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"88d0baaeb3ce553af8f103da0477073c87b8bd2e014b51494b3e15118a9a1525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"65b5dbb9967304eedefd97b212163b3f8af9b0b2bb0188919a3d12f3a6538fc2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de254d309520eb7dcc15cb4c173bd21c1b68dfea356b64131ea7e4a21eaaa4c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","failed_at":"e2e","signature_id":"524cdf887bee162758b9f6eca5510d2578bb6ecf654e9948629c9452eaf1782b","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100913554960945152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"open","pr_sha":"acfa8be9cc45d0569ea8da4e90158944e7541735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T11:43:14Z","started_at":"2026-09-18T11:43:14Z","completed_at":"2026-09-18T13:42:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.113325155Z","timing_metadata_checked_at":"2026-09-18T15:41:07.113325155Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.11742317Z","region_metadata_checked_at":"2026-09-18T15:41:07.11742317Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100898990957531136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6694,"pr_state":"closed","pr_sha":"f1437e7522fb6efcd77f296c45896ba121660db7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T10:45:26Z","started_at":"2026-09-18T10:45:26Z","completed_at":"2026-09-18T10:57:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:18.538824492Z","timing_metadata_checked_at":"2026-09-18T15:40:18.538824492Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:18.542726706Z","region_metadata_checked_at":"2026-09-19T03:30:33.749012857Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"159e2b1150b1ec7c3f974e57cd7712f7f278dc6df04ca2922e456ef7b1b80ed7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100898990957531136","occurred_at":"2026-09-18T10:45:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100892388036710400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6879,"pr_state":"closed","pr_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","final_merged_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T10:19:12Z","started_at":"2026-09-18T10:19:12Z","completed_at":"2026-09-18T12:27:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.766218983Z","timing_metadata_checked_at":"2026-09-18T15:40:29.766218983Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.769670695Z","region_metadata_checked_at":"2026-09-18T15:40:29.769670695Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"91261d5b93a8d4039291003288ddae318a2a8506","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:00:26Z","started_at":"2026-09-18T10:00:26Z","completed_at":"2026-09-18T12:04:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.436005317Z","timing_metadata_checked_at":"2026-09-18T15:41:20.436005317Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.439499329Z","region_metadata_checked_at":"2026-09-18T15:41:20.439499329Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"43a0402625e5406ea76d4f94e1c32004477778ea8ad7365eb990e95245f7b6c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","occurred_at":"2026-09-18T10:00:26Z","failed_at":"alert","signature_id":"c3bce50243ff744e90a9975f976bf7851614b19682c44e007ffa00d7c6f101f9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:02:41Z\n Ended: 2026-09-18T11:23:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5658496-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stcj8snr7hlhv8hfthg1emt8dhhfded-j1a6w4y5x8t0x1d\", pod=\"router-769f87966d-khp89\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stcj8snr7hlhv8hfthg1emt8dhhfded-j1a6w4y5x8t0x1d/router-769f87966d-khp89 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T11:04:44Z\n Ended: 2026-09-18T11:23:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5658496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stckihvk61lv7eojs5hbnlalilj7v1i-agg-cluster\", pod=\"router-7c9ffb6845-wczwq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stckihvk61lv7eojs5hbnlalilj7v1i-agg-cluster/router-7c9ffb6845-wczwq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T09:55:36Z","started_at":"2026-09-18T09:55:36Z","completed_at":"2026-09-18T12:07:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.702185479Z","timing_metadata_checked_at":"2026-09-18T15:41:15.702185479Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.706062693Z","region_metadata_checked_at":"2026-09-18T15:41:15.706062693Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"778d8ce6a0c00a1e3bda2fb387049c59cc3d888b0904fe2e8c3e42f76d242cb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","occurred_at":"2026-09-18T09:55:36Z","failed_at":"alert","signature_id":"fcdb5ce6d5eb5426a483be7faf7e19f591fbf253b3ebf7b8ca13d6e2fd0d11f4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:09:41Z\n Ended: 2026-09-18T11:20:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9620352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stclp139sl99taqsqoij7cie2hd4cpk-v20260901\", pod=\"router-5b84bc6565-scjwf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stclp139sl99taqsqoij7cie2hd4cpk-v20260901/router-5b84bc6565-scjwf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T09:52:14Z","started_at":"2026-09-18T09:52:14Z","completed_at":"2026-09-18T12:14:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.781146879Z","timing_metadata_checked_at":"2026-09-18T15:40:08.781146879Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:08.784934493Z","region_metadata_checked_at":"2026-09-18T15:40:08.784934493Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"6926365da1644e2f91dfd7c5fbe3a79ab84cf7c2c913a442441f97c35df1ad0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","failed_at":"alert","signature_id":"4e84895295e21c9ae798ddae187f381094acb88d5bb19172060995a42efdfbcd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:21:18Z\n Ended: 2026-09-18T11:37:18Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3790208-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3790208-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"da069b10122f51fa80e0ae2bcf494afdef048174e2a89a3b93080f1a41097cda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","failed_at":"alert","signature_id":"002a8aff34d1219bae02653dd6f6335a032c8c53538dc0e88e47e06a4653de19","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T11:14:28Z\n Ended: 2026-09-18T11:23:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3790208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stcn6svivskocqagj4qaaqujiq0cekp-m1e6y0x7h1c1n8p\", pod=\"router-748fddb496-fcbbx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stcn6svivskocqagj4qaaqujiq0cekp-m1e6y0x7h1c1n8p/router-748fddb496-fcbbx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T09:21:30Z","started_at":"2026-09-18T09:21:30Z","completed_at":"2026-09-18T11:21:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.520475064Z","timing_metadata_checked_at":"2026-09-18T15:40:45.520475064Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.524059977Z","region_metadata_checked_at":"2026-09-18T15:40:45.524059977Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"aa9e1643ab2e037088a071c942816832f5dd1d5c40161259238d64e030d67554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"alert","signature_id":"7eefe8fce4efeddab6a7bddfd5f7e5cb50289671f56d6e25708c25b9305ce796","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T10:18:52Z\n Ended: 2026-09-18T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8771328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbuceofnif99sq6etloedbqkpc5qh9-u1a3e7t8x9z9b1l\", pod=\"router-b6965d7dc-jzrb7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbuceofnif99sq6etloedbqkpc5qh9-u1a3e7t8x9z9b1l/router-b6965d7dc-jzrb7 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"d5dea3392eb2163b7d8f1d40e9615b3d55175ba5913ecb983521f10c10fa03a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"alert","signature_id":"077ee51cbdfb7388e2c8c28578e14f7c26aad49cf417b4ab9a95eda6e094ce1d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T10:27:52Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T10:28:56Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"fdd4ba74c769ebfd11719783a6a41b89c38eea91e67ee842d218c8207011b514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","failed_at":"e2e","signature_id":"f590a6d9decf3d5d50d5dfcef242c5c8af6ebdbb758d3e90d6fae5136781bae3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:44:01Z","started_at":"2026-09-18T08:44:01Z","completed_at":"2026-09-18T10:35:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.135552482Z","timing_metadata_checked_at":"2026-09-18T15:40:44.135552482Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.138715793Z","region_metadata_checked_at":"2026-09-18T15:40:44.138715793Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"955bb3878fbb804e63ad31f56c01fd2e666068f98c58de7ff1ea549e8b5da0aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"alert","signature_id":"be273c1646946db878b565bca63154b691913899e238a33aee949d4d46ae8e4f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:36:00Z\n Ended: 2026-09-18T10:18:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3745536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stba81rimklhsg3cr5ktr5tf749cko1-u3d9c8l0y5d3t2z\", pod=\"router-5b649bbbf7-4cvr4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stba81rimklhsg3cr5ktr5tf749cko1-u3d9c8l0y5d3t2z/router-5b649bbbf7-4cvr4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"985a99cacb656894b6047e4124ce83ff8d4d737c41dac3e33a006cde078ecdf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"78800d7f9bd7ce60d82097401bbc1a1b346c5eb770d7eeedc635c6d48abac870","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-l7rx685m27vp/cluster-candidate-4-20-4l4f7q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ac60\u003e: \n failed to create HCP cluster cluster-candidate-4-20-4l4f7q: failed waiting for cluster=\"cluster-candidate-4-20-4l4f7q\" in resourcegroup=\"rg-candidate-4-20-l7rx685m27vp\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"name\": \"d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:24:34.52165977Z\",\n \"endTime\": \"2026-09-18T09:53:41.437046134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 21m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-l7rx685m27vp/cluster-candidate-4-20-4l4f7q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ac60\u003e: \n failed to create HCP cluster cluster-candidate-4-20-4l4f7q: failed waiting for cluster=\"cluster-candidate-4-20-4l4f7q\" in resourcegroup=\"rg-candidate-4-20-l7rx685m27vp\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"name\": \"d0d44620-2f5c-46e9-97e5-7104c5ac9593\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:24:34.52165977Z\",\n \"endTime\": \"2026-09-18T09:53:41.437046134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 21m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c570e2e00a3bb9b3ebf76b01680e0fe0c751b734150705cdfbf1f0aa48f7c616","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"ba73d1a54ec8e3a0b02523dbd4276ff942ead7e6f9b4e52703889f225a290272","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000faea40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdl2drfvfhtk\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\",\n \"name\": \"4fc5c348-7117-4850-997b-20eb599df30e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:25:07.592750123Z\",\n \"endTime\": \"2026-09-18T09:54:11.523791716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000faea40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdl2drfvfhtk\" to finish creating: GET https://rp.j3745536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc5c348-7117-4850-997b-20eb599df30e\",\n \"name\": \"4fc5c348-7117-4850-997b-20eb599df30e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:25:07.592750123Z\",\n \"endTime\": \"2026-09-18T09:54:11.523791716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0535b166f31d721ee73bca028f5c2a263e713a6075f4c754234b073b9f48232","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"open","pr_sha":"acfa8be9cc45d0569ea8da4e90158944e7541735","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:38:28Z","started_at":"2026-09-18T08:38:28Z","completed_at":"2026-09-18T10:37:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.016824507Z","timing_metadata_checked_at":"2026-09-18T15:41:07.016824507Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.022435828Z","region_metadata_checked_at":"2026-09-18T15:41:07.022435828Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2423d0d06a02e9c0ba4df730fd696a975e13e7e397a1f473b152ad31b5e16db0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"alert","signature_id":"d6db3f97bdaa9f3b8874f62b7846be7bdd694b5e229a6dfe93f7bd212a0dcb02","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:41:54Z\n Ended: 2026-09-18T10:11:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5368576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbcs84sip93t6dffm9229aerut2acv-z8r9c8z3p9s9m1i\", pod=\"router-d76b55d7-rgg2v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbcs84sip93t6dffm9229aerut2acv-z8r9c8z3p9s9m1i/router-d76b55d7-rgg2v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T09:42:48Z\n Ended: 2026-09-18T10:11:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5368576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stbd564riaqku802qrnf0q9bc8r9vik-private-kas\", pod=\"router-85dc58d9b7-tsdrp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stbd564riaqku802qrnf0q9bc8r9vik-private-kas/router-85dc58d9b7-tsdrp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cb9dd419229483ed402cd42b90130ff4f3bbfbe90e59cf993cdccf94f29ee124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"e2e","signature_id":"bb9a6198465bbe86bad7f0728097a5e8139953db856c6c8aad613b2d535d245f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcaa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5jssdfr4s6w\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"name\": \"92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:20.728673075Z\",\n \"endTime\": \"2026-09-18T09:58:22.849626679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcaa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5jssdfr4s6w\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"name\": \"92cf1c4a-bb92-46ae-81a1-0fe741329a13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:20.728673075Z\",\n \"endTime\": \"2026-09-18T09:58:22.849626679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d91a2664b239a950a30ac6a20b3487dd4fbe89eeb800e668ff0ded3b40618f20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","failed_at":"e2e","signature_id":"2e53b9a5382ed0f0493cfc0c48e9bf7aa226835370880aa8b0e3c984f3d43f8d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8kv74t8cs9l5\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"name\": \"4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:55.514391889Z\",\n \"endTime\": \"2026-09-18T09:59:02.761026905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8kv74t8cs9l5\" to finish creating: GET https://rp.j5368576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"name\": \"4ec06138-7d7f-447e-acf1-59e926e58e3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:29:55.514391889Z\",\n \"endTime\": \"2026-09-18T09:59:02.761026905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863236734717952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T08:23:22Z","started_at":"2026-09-18T08:23:22Z","completed_at":"2026-09-18T10:34:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.989207567Z","timing_metadata_checked_at":"2026-09-18T15:40:34.989207567Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.992552479Z","region_metadata_checked_at":"2026-09-18T15:40:34.992552479Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"8c23f9b0c37e8f10e070fd2c1d0831ee8fd99e56","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:23:07Z","started_at":"2026-09-18T08:23:07Z","completed_at":"2026-09-18T10:24:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:39.485293545Z","timing_metadata_checked_at":"2026-09-18T15:40:39.485293545Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:39.489133258Z","region_metadata_checked_at":"2026-09-18T15:40:39.489133258Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2028dd15cd9a1813f35b867a2d1aed5557176f389b8b74009f14759aed2232e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"1ff84f075d12052118fcca1c651ed549b6c5f0208069e0615f46732cae26b19e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp"}},{"row_id":"2be3e643fa3f57bb4de7eb40d271fd16091009ad89f2a29e90e00b3230c7852c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"7193ae4d62017e53f8c9422389a5405738a2e8a036e8555e76291f330d89920e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"8e6e197a9c1d0541b3d16ee31945ea91c35418b1bc264b205fca4968923fafec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:27:59Z\n Ended: 2026-09-18T09:34:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6818048-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9c239768e853cd4f0bde3ead855329c5aa91e04279ccb76ac5426e3a965ee90f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"alert","signature_id":"f47f0b6edf615e207ea20e3174c9c4b6c3860a8cd322039375d46696bfbe5705","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:24:01Z\n Ended: 2026-09-18T09:49:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6818048-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stb4ma2bs7029nmfb05hiqpcdsgabul-w1y0u6t1v1n3z9l\", pod=\"router-6bd77b545b-pvt7r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stb4ma2bs7029nmfb05hiqpcdsgabul-w1y0u6t1v1n3z9l/router-6bd77b545b-pvt7r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e43e42f4a6d0e49318031ab937af0f655181064f910d931a061226af618cc553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","failed_at":"e2e","signature_id":"001338be47cf8dcaa3fc3c500d6a6dec5a631054e1ca9f9d544c49d235b2dcb7","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9a600\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pwncww98jzm8\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"name\": \"145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:11:52.011321272Z\",\n \"endTime\": \"2026-09-18T09:40:57.964346748Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9a600\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pwncww98jzm8\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"name\": \"145961c0-6282-4bf7-a47e-98a2a7475ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:11:52.011321272Z\",\n \"endTime\": \"2026-09-18T09:40:57.964346748Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863036339261440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:22:32Z","started_at":"2026-09-18T08:22:32Z","completed_at":"2026-09-18T08:23:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.001130298Z","timing_metadata_checked_at":"2026-09-18T15:40:44.001130298Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:44.004775711Z","region_metadata_checked_at":"2026-09-19T03:30:37.157285126Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2e6f85effdaeb3112437499df2f3a95965bc9163b1e03939a870a0232f47889c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863036339261440","occurred_at":"2026-09-18T08:22:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T08:21:29Z","started_at":"2026-09-18T08:21:29Z","completed_at":"2026-09-18T10:18:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.318614845Z","timing_metadata_checked_at":"2026-09-18T15:41:34.318614845Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.321862257Z","region_metadata_checked_at":"2026-09-18T15:41:34.321862257Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6c210d1a83ddb761e4bfb2fba935a2a001fb9c36576fa71840fc19636c6d912b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","occurred_at":"2026-09-18T08:21:29Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:23:28Z\n Ended: 2026-09-18T09:39:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8030592-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7003,"pr_state":"open","pr_sha":"5ef6466e2a8ec2041b8b57cd5821ccdc91bbdd88","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T08:07:05Z","started_at":"2026-09-18T08:07:05Z","completed_at":"2026-09-18T09:54:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:22.074694851Z","timing_metadata_checked_at":"2026-09-18T15:41:22.074694851Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:22.077840163Z","region_metadata_checked_at":"2026-09-18T15:41:22.077840163Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f2b83989eb95efb2403df14346d44f01552bcb574650cc9c2db78d28140feaec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","occurred_at":"2026-09-18T08:07:05Z","failed_at":"e2e","signature_id":"4553dd78f39dc989ce886fb5e82604c7aa8be953927622430a5564e21a9cc7f0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100856808020119552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6962,"pr_state":"closed","pr_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","final_merged_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T07:57:47Z","started_at":"2026-09-18T07:57:47Z","completed_at":"2026-09-18T09:57:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.306908853Z","timing_metadata_checked_at":"2026-09-18T15:40:58.306908853Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.310468766Z","region_metadata_checked_at":"2026-09-18T15:40:58.310468766Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T06:57:25Z","started_at":"2026-09-18T06:57:25Z","completed_at":"2026-09-18T09:16:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.993207072Z","timing_metadata_checked_at":"2026-09-18T15:40:40.993207072Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.996500084Z","region_metadata_checked_at":"2026-09-18T15:40:40.996500084Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ee4aebdad12a21a86688d0b3660ead46e494170673ae2c2e6e13761cebb14f76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","occurred_at":"2026-09-18T06:57:25Z","failed_at":"alert","signature_id":"0d95d8184561f593338e86b1c11b5ad3f6442e5263869a130eac56ca5e35b079","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:31:06Z\n Ended: 2026-09-18T08:38:05Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5294976-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5294976-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T06:57:14Z","started_at":"2026-09-18T06:57:14Z","completed_at":"2026-09-18T09:19:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.949440129Z","timing_metadata_checked_at":"2026-09-18T15:41:12.949440129Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.952906042Z","region_metadata_checked_at":"2026-09-18T15:41:12.952906042Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"5d8b5e5c733e92bb36be533024a94ed93a2579559c695da763da987f7d7bdb48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","failed_at":"alert","signature_id":"b8ef7bc40a5e42cb1d9a25c2c491ed35aa8060746d13168cf09c7ed965a89066","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:33:35Z\n Ended: 2026-09-18T08:39:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4018048-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4018048-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fdfc1f8ee548108afcd7a729dce171c7ad549661d33c82f685b2b89ceeabf469","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","failed_at":"alert","signature_id":"7b986d6b9e982cdf65d80f902e9563f9d1125a3f54f3c1a12116db416ae5f10d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T08:44:25Z\n Ended: 2026-09-18T08:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4018048-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sta451luu6i39gfi3s8vomm21o2b1lj-v1u6q6s5g4j0h3p\", pod=\"kube-apiserver-5959d9c69f-k9pqp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sta451luu6i39gfi3s8vomm21o2b1lj-v1u6q6s5g4j0h3p/kube-apiserver-5959d9c69f-k9pqp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100833415631736832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"a20dd280dd6b7499cf853051219487314ef1e460","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:24:52Z","started_at":"2026-09-18T06:24:52Z","completed_at":"2026-09-18T08:23:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.356877104Z","timing_metadata_checked_at":"2026-09-18T15:40:31.356877104Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:31.360236816Z","region_metadata_checked_at":"2026-09-18T15:40:31.360236816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c0ebd76b50205ad901e577ade460296f93b4079e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T06:09:33Z","started_at":"2026-09-18T06:09:33Z","completed_at":"2026-09-18T08:14:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.906877307Z","timing_metadata_checked_at":"2026-09-18T15:41:01.906877307Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.909999818Z","region_metadata_checked_at":"2026-09-18T15:41:01.909999818Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"058d4d1662f3ff9282ca6e8faa2c2cae8d2bf7201d772467499d9a0089725839","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"79534399cf979be34c0ffb48ddc762d6841c4a16f58dc222924b468f595a7adb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ff8902bcd442597127933fe639d5a91a9e1d31e98c38e00f7d9192969447637","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"abcb5fca9b88f5de84bce4c63659906c7070748eea55c112da30810936b0fc97","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-895tpwlfvzpg/cluster-candidate-5-0-jpmgmm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e760\u003e: \n failed to create HCP cluster cluster-candidate-5-0-jpmgmm: failed waiting for cluster=\"cluster-candidate-5-0-jpmgmm\" in resourcegroup=\"rg-candidate-5-0-895tpwlfvzpg\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"name\": \"d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:57:19.894043957Z\",\n \"endTime\": \"2026-09-18T07:26:23.420290614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-895tpwlfvzpg/cluster-candidate-5-0-jpmgmm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e760\u003e: \n failed to create HCP cluster cluster-candidate-5-0-jpmgmm: failed waiting for cluster=\"cluster-candidate-5-0-jpmgmm\" in resourcegroup=\"rg-candidate-5-0-895tpwlfvzpg\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"name\": \"d838caae-4be9-4361-b2b6-0820103db4f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:57:19.894043957Z\",\n \"endTime\": \"2026-09-18T07:26:23.420290614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24e519c9ea99624bed46d72420daea446f43d0f46833aed446771e34255ebefd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"cc794ab286f7ef163ad70de687149ee27cd59dcc48d24d280de823796404177d","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"45eb23e248127a5fd1cf73f5a23034867813c0e52e1e7dd6e30bab10faa48bb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"782ec114fda729c308336015b9ba3ea7dba7299080a2ca46beda618ff211a392","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080af80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4pnfzqpsrckd\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"name\": \"540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:58:33.125357484Z\",\n \"endTime\": \"2026-09-18T07:27:33.397040852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080af80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4pnfzqpsrckd\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"name\": \"540f69ae-a13e-46ee-bf0d-4de42d2cfa88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:58:33.125357484Z\",\n \"endTime\": \"2026-09-18T07:27:33.397040852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"af42e922170b2d99c9383756f170a9227b8fb5aaf3e36571cffd112ac2677d29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","failed_at":"e2e","signature_id":"8db752092d73d2ebfaaaef7b1150f759438bf639db4c2c7645ebdce228946dcf","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100828394844327936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:04:55Z","started_at":"2026-09-18T06:04:55Z","completed_at":"2026-09-18T08:14:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.536925977Z","timing_metadata_checked_at":"2026-09-18T15:40:59.536925977Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.539776288Z","region_metadata_checked_at":"2026-09-18T15:40:59.539776288Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827927296872448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"96f38fd5a277adea0aeec323ef5685124af19024","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T06:03:03Z","started_at":"2026-09-18T06:03:03Z","completed_at":"2026-09-18T08:09:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.149508965Z","timing_metadata_checked_at":"2026-09-18T15:40:29.149508965Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.153306579Z","region_metadata_checked_at":"2026-09-18T15:40:29.153306579Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T06:01:39Z","started_at":"2026-09-18T06:01:39Z","completed_at":"2026-09-18T07:55:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.162062581Z","timing_metadata_checked_at":"2026-09-18T15:41:34.162062581Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.165221793Z","region_metadata_checked_at":"2026-09-18T15:41:34.165221793Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"19cc5011eb4bf8fc2743655dddceed3860753b855db9569c6b609dac39300849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"e2e","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"510d2ef135f1ec6cd6b28e6a8dff9c37612315d8ac546638abd6539fccc9f0e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"e2e","signature_id":"25746549fcd7e3d493a2e9010aa4c2f674167bdbce01dd388ff30ebecae482d7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"96d68ed0e6539b9919596989a03ac86d6279a7ffed0bce91ea23ad81d6388996","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","failed_at":"alert","signature_id":"56136cc3eeacd2663934d686bb2d9b29fb8b76aaa1ea7443d05087ccb6f12b10","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:59:52Z\n Ended: 2026-09-18T07:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91kv9p4h3i5f2qjatf1ddh7hg451l-u6e3o9h6n4t0q1m\", pod=\"router-56dbc9cf68-flvzm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91kv9p4h3i5f2qjatf1ddh7hg451l-u6e3o9h6n4t0q1m/router-56dbc9cf68-flvzm has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T07:00:48Z\n Ended: 2026-09-18T07:18:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91ihbjq7vgovat1v96s2dgh6gc8k8-ingress-cert-le\", pod=\"router-866f65df77-vw5dl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91ihbjq7vgovat1v96s2dgh6gc8k8-ingress-cert-le/router-866f65df77-vw5dl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-18T07:00:48Z\n Ended: 2026-09-18T07:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7100672-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st91vs1vhcp81dngelnmuif63cd1p56-private-kas\", pod=\"router-6449dfbd8c-5w442\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st91vs1vhcp81dngelnmuif63cd1p56-private-kas/router-6449dfbd8c-5w442 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100825645108957184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:53:59Z","started_at":"2026-09-18T05:53:59Z","completed_at":"2026-09-18T06:03:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.572008709Z","timing_metadata_checked_at":"2026-09-18T15:41:15.572008709Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:15.57529412Z","region_metadata_checked_at":"2026-09-19T03:30:40.600884587Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"22e97c2e288eaaab62face32d60d489ff031945e6aa062cea0ce6394812e8872","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100825645108957184","occurred_at":"2026-09-18T05:53:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6962,"pr_state":"closed","pr_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","final_merged_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:20:59Z","started_at":"2026-09-18T05:20:59Z","completed_at":"2026-09-18T07:10:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.18925523Z","timing_metadata_checked_at":"2026-09-18T15:40:58.18925523Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.192641342Z","region_metadata_checked_at":"2026-09-18T15:40:58.192641342Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"48fc808570ba8478c7da5f5a570807c3fc61e66dfbdc6c290988b25eed84bb9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","occurred_at":"2026-09-18T05:20:59Z","failed_at":"e2e","signature_id":"fd810f6089fd17b9ae89f5ca382f4365705ce74f6259d695bca5e674d7783e53","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817174481342464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:20:20Z","started_at":"2026-09-18T05:20:20Z","completed_at":"2026-09-18T06:09:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.802472898Z","timing_metadata_checked_at":"2026-09-18T15:41:12.802472898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.812156633Z","region_metadata_checked_at":"2026-09-18T15:41:12.812156633Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ccbfb930772320d5e8f6f15fc7bd93741f16a645865155f636c41269986ea602","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817174481342464","occurred_at":"2026-09-18T05:20:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6813,"pr_state":"open","pr_sha":"4a5aea6cd3a0485cade670b99e2a3f7a2e59ebbb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:13:05Z","started_at":"2026-09-18T05:13:05Z","completed_at":"2026-09-18T07:09:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.033420457Z","timing_metadata_checked_at":"2026-09-18T15:40:25.033420457Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.036653268Z","region_metadata_checked_at":"2026-09-18T15:40:25.036653268Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"74ed0f71392d5582daeceec0dc12cc85e38ab0e9c2693ba7191fa2ed8345e536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","failed_at":"e2e","signature_id":"1b937eb2ed1eae4fc31ae0598f8de6ce2076a500845fc3ea7dcd762870526b2e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0009cb668\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6k7xjqv9k7sk\" to finish creating: GET https://rp.j1065216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"name\": \"6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:12:51.736703148Z\",\n \"endTime\": \"2026-09-18T06:31:55.141914948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0009cb668\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6k7xjqv9k7sk\" to finish creating: GET https://rp.j1065216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"name\": \"6b3fe34c-12d6-487f-9607-688763ac04db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:12:51.736703148Z\",\n \"endTime\": \"2026-09-18T06:31:55.141914948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e3cf9384daecc3451f8c4882fa99d01a23436abc728c976e75e44526ae05e9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","failed_at":"e2e","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6157,"pr_state":"open","pr_sha":"d4af5249d6e0478ea45dc795343aaa530c0f147b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:12:47Z","started_at":"2026-09-18T05:12:47Z","completed_at":"2026-09-18T07:19:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.885570852Z","timing_metadata_checked_at":"2026-09-18T15:40:04.885570852Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.888765463Z","region_metadata_checked_at":"2026-09-18T15:40:04.888765463Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1d2ec301c869b7058e1550ed7eef64aa4c80e53281a61a75e279669c24f17ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"4fb56cfa0c5d81c59f21dba90bd2b4aeaa2b18f5202205163a7a19b83e403f4e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2238aef8fd05ba5d9f163445be374775a863bb390c7296dbc2733ec0a60e5e06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"alert","signature_id":"49184b986d87df135df2fc3dcc64465876a9918af7c03bd147e1ea6126eccebf","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:22:09Z\n Ended: 2026-09-18T06:28:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0748928-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0748928-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"31bdad879e96505d0a8491d793dba34142cef8ede9e4f2fcd6b3ae74df94b79f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"alert","signature_id":"476aed8123ee528db23f3663c1332194314ed537bec78310ea649964cd6c166f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T07:08:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0748928-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9hl75\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9hl75 (namespace arobit) on cluster ci01-j0748928-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"7a969b98fae0731b36cb6bf2edab5476bfc770f0148e570c222bfc1fc55de8d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"0d8835203476a034717107aadec991b28272c94910dad48bde3bb462bb56ac9b","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bf9085c5865b518467942404d4892e807542a93a01922bb60f84e377306713e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","failed_at":"e2e","signature_id":"06ba3d2a9582d2a18869bdd5ae1df4321a4d0977a7be4bdd0e3cc1af2ec0608a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","failure_pattern":"\"error\": {","search_query":"\"error\": {"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815135416258560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"728300dde5778eae2cc566e635f9026903fc274b","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:12:14Z","started_at":"2026-09-18T05:12:14Z","completed_at":"2026-09-18T05:35:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.429124592Z","timing_metadata_checked_at":"2026-09-18T15:41:15.429124592Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.440679834Z","region_metadata_checked_at":"2026-09-18T15:41:15.440679834Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c1000a36e27a701f2160dcd785daa85c641b7ac04049d77fb5874c75fb871f9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815135416258560","occurred_at":"2026-09-18T05:12:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815091837440000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T05:12:03Z","started_at":"2026-09-18T05:12:03Z","completed_at":"2026-09-18T07:06:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.171413155Z","timing_metadata_checked_at":"2026-09-18T15:40:24.171413155Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:24.177127876Z","region_metadata_checked_at":"2026-09-18T15:40:24.177127876Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T05:01:15Z","started_at":"2026-09-18T05:01:15Z","completed_at":"2026-09-18T07:06:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.638509365Z","timing_metadata_checked_at":"2026-09-18T15:40:08.638509365Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:08.642040978Z","region_metadata_checked_at":"2026-09-18T15:40:08.642040978Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"fac7c2814ab8e43eaa9ded8dba266bbff2ce61f4f6d45594d0fd1ccf378fdfd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","occurred_at":"2026-09-18T05:01:15Z","failed_at":"alert","signature_id":"e85df63033962c2849bfc318fc85b0a8cba9948dc2466a5fa45b6b2724994bfb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T06:00:50Z\n Ended: 2026-09-18T06:20:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2458624-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st858a4pvmlmac4uvehhfkife65pian-u9l5q3l8s3c3d7s\", pod=\"router-d9f7df5f5-npvv8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st858a4pvmlmac4uvehhfkife65pian-u9l5q3l8s3c3d7s/router-d9f7df5f5-npvv8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100811166501572608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T04:56:27Z","started_at":"2026-09-18T04:56:27Z","completed_at":"2026-09-18T07:01:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:47.107173571Z","timing_metadata_checked_at":"2026-09-18T15:40:47.107173571Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:47.110384283Z","region_metadata_checked_at":"2026-09-18T15:40:47.110384283Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100794344981139456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6930,"pr_state":"closed","pr_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","final_merged_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T03:49:37Z","started_at":"2026-09-18T03:49:37Z","completed_at":"2026-09-18T05:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:34.041950448Z","timing_metadata_checked_at":"2026-09-18T15:41:34.041950448Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:34.045422161Z","region_metadata_checked_at":"2026-09-18T15:41:34.045422161Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T03:21:45Z","started_at":"2026-09-18T03:21:45Z","completed_at":"2026-09-18T05:18:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.085262804Z","timing_metadata_checked_at":"2026-09-18T15:40:40.085262804Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.088495616Z","region_metadata_checked_at":"2026-09-18T15:40:40.088495616Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"3e65bdb606fff0e5981bdba342a4791f3d2fa1e38c8f50c107ed42a823bbe88c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"e2e","signature_id":"89409b5febc6093a9cefc9b6ca32afd019b45b211957e3bd23fe77c3eda373f3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93fc43fa475e4e277bb2bc7fad23f329306323c4ccc6329418ef90279c7fad2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"b14a5290094a9e0d25292157800f9158e06c44b980a0e58131fba650ea86837b","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:31:05Z\n Ended: 2026-09-18T04:48:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e333cab1-7b1b-43d4-a2a8-c38ce512c9fe\"\n Description: Pod ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"a32bb23b9959f8e7fb533a763afb0ee6f3f0488034bca1a4c24857e25b84f5e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:18:48Z\n Ended: 2026-09-18T04:37:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2362240-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"aa045988aa28993d83b618eef4ca84ea8647d0e81961c1247861c45b4948742d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","failed_at":"alert","signature_id":"32caccfe24784ea73efa24f662aed9cbdd2ff753b7f3457b2fae68c38d3d3c3d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:29:08Z\n Ended: 2026-09-18T04:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2r9f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T04:29:08Z\n Ended: 2026-09-18T04:55:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2r9f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6984,"pr_state":"closed","pr_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","final_merged_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:11:35Z","started_at":"2026-09-18T03:11:35Z","completed_at":"2026-09-18T03:46:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.171007071Z","timing_metadata_checked_at":"2026-09-18T15:41:09.171007071Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.174145883Z","region_metadata_checked_at":"2026-09-18T15:41:09.174145883Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bfe862bede5f150ebf4ee6361d1b091eea5de565399b30f38ee0007fd9f2c344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","occurred_at":"2026-09-18T03:11:35Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-18T03:37:37.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T03:37:38.393Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-18T03:37:39.053Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1401344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298\ntime=2026-09-18T03:40:39.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298/operationStatuses/08584119042267536349\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-sr7huchijtjyg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1401344-mgmt-1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6984,"pr_state":"closed","pr_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","final_merged_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:08:52Z","started_at":"2026-09-18T03:08:52Z","completed_at":"2026-09-18T05:05:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.915554293Z","timing_metadata_checked_at":"2026-09-18T15:41:33.915554293Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.918892505Z","region_metadata_checked_at":"2026-09-18T15:41:33.918892505Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"3d70f2e86360715a46ec9ffc99c890cec9d30c215df5fed8e628acfbceddd40b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"alert","signature_id":"4750aa456ebf384aa187370c588c7897567d97246241f2ecee3733d106d2cd85","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:02:41Z\n Ended: 2026-09-18T04:32:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6907648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6e0gt56ef7jnfg7trhv4ir2638kjr-n2s7g5a5a5z9k1l\", pod=\"router-7d99594d96-6l44r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6e0gt56ef7jnfg7trhv4ir2638kjr-n2s7g5a5a5z9k1l/router-7d99594d96-6l44r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"788ccc521a93ece0184485b0fec366b00c6d4b00fc52518c2edc249a9588e08a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"a2d0482738306402a9aee374779b77d199fad57d181dc39a001fc3fa8cf50f3a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-8rxkwjmg9fdn/cluster-candidate-4-22-8z2p5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017682a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8z2p5b: failed waiting for cluster=\"cluster-candidate-4-22-8z2p5b\" in resourcegroup=\"rg-candidate-4-22-8rxkwjmg9fdn\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"name\": \"a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.402664583Z\",\n \"endTime\": \"2026-09-18T04:19:53.994096677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-8rxkwjmg9fdn/cluster-candidate-4-22-8z2p5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017682a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8z2p5b: failed waiting for cluster=\"cluster-candidate-4-22-8z2p5b\" in resourcegroup=\"rg-candidate-4-22-8rxkwjmg9fdn\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"name\": \"a7ca75c5-62a9-48f1-8ad3-c65e9f548e4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.402664583Z\",\n \"endTime\": \"2026-09-18T04:19:53.994096677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5551cdba1db795af316c870fe01077a6171db4f39637bb55efbc729a8f979fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"82775d4e1d46d1566b3f4eab5619ed61b625107edb899f3a6d4e9d4864e726a1","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--lv2mjp29lzrs/cluster-hs-pre-xxp2zq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aef7c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xxp2zq: failed waiting for cluster=\"cluster-hs-pre-xxp2zq\" in resourcegroup=\"rg-hs-pre--lv2mjp29lzrs\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\",\n \"name\": \"da216ffc-9136-436b-b01d-92158b449e7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:26.310458979Z\",\n \"endTime\": \"2026-09-18T04:19:33.969303507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--lv2mjp29lzrs/cluster-hs-pre-xxp2zq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aef7c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xxp2zq: failed waiting for cluster=\"cluster-hs-pre-xxp2zq\" in resourcegroup=\"rg-hs-pre--lv2mjp29lzrs\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da216ffc-9136-436b-b01d-92158b449e7c\",\n \"name\": \"da216ffc-9136-436b-b01d-92158b449e7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:26.310458979Z\",\n \"endTime\": \"2026-09-18T04:19:33.969303507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb3120bfdce23eefa3b09de56c92ea4981f8b18405e705990a83a0a96ae4d4b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"e2e","signature_id":"925fc6ad3116b3a4af4d42c985323051e3b386c99d2bbff7d1917ed9c6b7ef67","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-wj65mtt7sqkj/cluster-candidate-4-21-pk5bmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181e300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pk5bmj: failed waiting for cluster=\"cluster-candidate-4-21-pk5bmj\" in resourcegroup=\"rg-candidate-4-21-wj65mtt7sqkj\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"name\": \"62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.884252443Z\",\n \"endTime\": \"2026-09-18T04:19:53.826763956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-wj65mtt7sqkj/cluster-candidate-4-21-pk5bmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181e300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pk5bmj: failed waiting for cluster=\"cluster-candidate-4-21-pk5bmj\" in resourcegroup=\"rg-candidate-4-21-wj65mtt7sqkj\" to finish creating: GET https://rp.j6907648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"name\": \"62f044a5-f392-4d7e-8a95-8aab4d322466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:50:46.884252443Z\",\n \"endTime\": \"2026-09-18T04:19:53.826763956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee75403d59abdaf958d8493407c062f18590022f6d80b33ca30dc123368aea31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:12:00Z\n Ended: 2026-09-18T04:22:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6907648-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100783081760755712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T03:04:51Z","started_at":"2026-09-18T03:04:51Z","completed_at":"2026-09-18T05:11:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.457172589Z","timing_metadata_checked_at":"2026-09-18T15:41:03.457172589Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:03.460414901Z","region_metadata_checked_at":"2026-09-18T15:41:03.460414901Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T03:01:52Z","started_at":"2026-09-18T03:01:52Z","completed_at":"2026-09-18T05:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.778959001Z","timing_metadata_checked_at":"2026-09-18T15:41:33.778959001Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.781922211Z","region_metadata_checked_at":"2026-09-18T15:41:33.781922211Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"45a2856401661fdd6bdb02772281f3f65800a3f67401bcdddeefa10e484edbb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"b838f89ca9ff0822f2ef510e9178d1d320d4033a5d5cd3ca5c98537110704efb","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001700260\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-hjgg7zlnkh76\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"name\": \"bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:43:00.850749816Z\",\n \"endTime\": \"2026-09-18T04:12:08.804551345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001700260\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-hjgg7zlnkh76\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"name\": \"bdfce236-4265-4d78-bac8-a4418dc005ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:43:00.850749816Z\",\n \"endTime\": \"2026-09-18T04:12:08.804551345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a901dd3c6c89997871f3068193d3cb8669ea78d44cff6f15a85a9ea4f76c0503","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"de20ef80e43125ed23aee61f6adcfb8942fa81561feaefb4e80cef19ed4db1c7","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]"}},{"row_id":"dbedf1be4eb49aa3f0b2032cb1641955f889c12ae3f4cb0b2cc59201a7dc52b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"6acf378b1d751cabdd2cb29f20e478c64231e5b033214257ad51d0d59a0c2ba7","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6746a4fee3e6eafc95c01f3459313ede8a21ef2bda7c7dcb1ccbc5b6d468dae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","failed_at":"e2e","signature_id":"f081e78216b7bd4a54e652f3dfd3a2cf8cf19638bb76bda01151759533a92078","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100781486385926144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:58:31Z","started_at":"2026-09-18T02:58:31Z","completed_at":"2026-09-18T02:59:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.619823827Z","timing_metadata_checked_at":"2026-09-18T15:41:33.619823827Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:33.622888838Z","region_metadata_checked_at":"2026-09-19T03:30:50.75322187Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8d22f66742ef131f38c7c4e93709705bbe4f7a01bdef73c50caa1c4619b06ca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100781486385926144","occurred_at":"2026-09-18T02:58:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:56:11Z","started_at":"2026-09-18T02:56:11Z","completed_at":"2026-09-18T05:00:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:38.618831126Z","timing_metadata_checked_at":"2026-09-18T15:40:38.618831126Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:38.62260664Z","region_metadata_checked_at":"2026-09-18T15:40:38.62260664Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0fff20b38be4d143d06a2cb3d086f989a5e9bbd8202944d8f44dc5f7fd4f21a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"e2e","signature_id":"c3a814dc2b1fc08b7b4fdb53010472ca317b4e82bd862888bcc25a0f8b4091ea","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-9ptnlfp98gkt/cluster-candidate-4-21-2b86tp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dac60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2b86tp: failed waiting for cluster=\"cluster-candidate-4-21-2b86tp\" in resourcegroup=\"rg-candidate-4-21-9ptnlfp98gkt\" to finish creating: GET https://rp.j2043264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"name\": \"60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:27.058951069Z\",\n \"endTime\": \"2026-09-18T04:15:29.848966802Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-9ptnlfp98gkt/cluster-candidate-4-21-2b86tp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dac60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2b86tp: failed waiting for cluster=\"cluster-candidate-4-21-2b86tp\" in resourcegroup=\"rg-candidate-4-21-9ptnlfp98gkt\" to finish creating: GET https://rp.j2043264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"name\": \"60784981-dab7-45ee-b0cb-ce3256c6fe9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:27.058951069Z\",\n \"endTime\": \"2026-09-18T04:15:29.848966802Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"307a452f46841553ddd43cbcf2b027eb5571187d7387898c99ad7aac211d3107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"e2e","signature_id":"66f62a5d5d02e16dfffeec66474c962d8c14552e10473a54125784861158f0d5","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","failure_pattern":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"\u003cvm\u003e\" failed and cleanup also failed: Get \"\u003curl\u003e\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"\u003curl\u003e\": stream error: stream ID 33; CANCEL; received from peer)","search_query":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)"}},{"row_id":"c232fee453456246aa92ab12966ed297d71c38a3433497408526219b347131fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","failed_at":"alert","signature_id":"a0cd1baa14d1b2c8e6eaa3868608b05d0234717e1c141c4d01a43194b1743c96","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T03:58:14Z\n Ended: 2026-09-18T04:08:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2043264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6c4ss9pg11n90barbef44mcqrigek-v20260901\", pod=\"router-85fc7f57f6-7t8xr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6c4ss9pg11n90barbef44mcqrigek-v20260901/router-85fc7f57f6-7t8xr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T04:00:16Z\n Ended: 2026-09-18T04:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2043264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st6c4u3st2ot5a3qc2a5p5fm6ceqr6n-t6k8m0v7e7f7n9m\", pod=\"router-5fbdb9d6b6-97rbb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st6c4u3st2ot5a3qc2a5p5fm6ceqr6n-t6k8m0v7e7f7n9m/router-5fbdb9d6b6-97rbb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:56:04Z","started_at":"2026-09-18T02:56:04Z","completed_at":"2026-09-18T05:18:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.380449661Z","timing_metadata_checked_at":"2026-09-18T15:40:45.380449661Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.383494872Z","region_metadata_checked_at":"2026-09-18T15:40:45.383494872Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"53b65b43a78d14c7c314b2fc2c97869861ce08c3f4d3ff086f45132c4267d786","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","occurred_at":"2026-09-18T02:56:04Z","failed_at":"e2e","signature_id":"3f4f2a1f835d80221855e8810686dfbc430d2c91ebb857edce8074813fea620c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T02:56:00Z","started_at":"2026-09-18T02:56:00Z","completed_at":"2026-09-18T05:05:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.849291554Z","timing_metadata_checked_at":"2026-09-18T15:40:40.849291554Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.852285665Z","region_metadata_checked_at":"2026-09-18T15:40:40.852285665Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"31f43db8ee5a53149901c5b44af4a35bf237df0e7455875f2880df11b194fafb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"993ed1b03e808393e377627c70564f00855cc6cb2d99bd1882ef2f7b7c73270e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vxdfqwnwx9d5/cluster-hs-pre-wvtqvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001020ba0\u003e: \n failed to create HCP cluster cluster-hs-pre-wvtqvz: failed waiting for cluster=\"cluster-hs-pre-wvtqvz\" in resourcegroup=\"rg-hs-pre--vxdfqwnwx9d5\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"name\": \"28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:19.328892293Z\",\n \"endTime\": \"2026-09-18T04:15:24.561215096Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vxdfqwnwx9d5/cluster-hs-pre-wvtqvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001020ba0\u003e: \n failed to create HCP cluster cluster-hs-pre-wvtqvz: failed waiting for cluster=\"cluster-hs-pre-wvtqvz\" in resourcegroup=\"rg-hs-pre--vxdfqwnwx9d5\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"name\": \"28738f7e-bf1e-435b-b7c4-b3ff10938c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:19.328892293Z\",\n \"endTime\": \"2026-09-18T04:15:24.561215096Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4027cb3b82230e7411762c2c3fcf16a5f39c47c82d42c9d2826a972e7806fc24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"a6058215e786bb088f449c6f95b45c11ce369a5297dbcc0542779a89a2c66add","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86e26915f7ac1662975a4eeffe0a1c22227598dded4b96a813ca69614d8e89fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"2e205df2fc76c59d7d5bced445a9af5fdbd8ffc0a4321fbfafdc448b6041ea0f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008963c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6htjbrlxsjhg\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"name\": \"b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:48.434685072Z\",\n \"endTime\": \"2026-09-18T04:15:54.641017549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m9s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008963c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6htjbrlxsjhg\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"name\": \"b82b6b34-867d-4037-91ad-ef65d0343bbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:48.434685072Z\",\n \"endTime\": \"2026-09-18T04:15:54.641017549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 15m9s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3a38d9eaaacffb8a3810d1f66e0b2e54b8a2e65a0fc51b92dba229525a06db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"a44cbebfe0aee283a67f51149529d8530cd9c5797c013288bce57d67f21ed3fb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dhdrlkcltwdt/cluster-candidate-5-0-tshnff should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d46700\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tshnff: failed waiting for cluster=\"cluster-candidate-5-0-tshnff\" in resourcegroup=\"rg-candidate-5-0-dhdrlkcltwdt\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\",\n \"name\": \"23603031-6375-471f-989c-9b2b7bce844f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:30.045260058Z\",\n \"endTime\": \"2026-09-18T04:15:34.702663145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 13m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dhdrlkcltwdt/cluster-candidate-5-0-tshnff should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d46700\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tshnff: failed waiting for cluster=\"cluster-candidate-5-0-tshnff\" in resourcegroup=\"rg-candidate-5-0-dhdrlkcltwdt\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23603031-6375-471f-989c-9b2b7bce844f\",\n \"name\": \"23603031-6375-471f-989c-9b2b7bce844f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:46:30.045260058Z\",\n \"endTime\": \"2026-09-18T04:15:34.702663145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 13m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a4d1a311e41b5620a81d14547a99e93448ca98bfb8e127aa350cd1bf0b63a0cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T02:55:52Z","started_at":"2026-09-18T02:55:52Z","completed_at":"2026-09-18T05:01:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.447320079Z","timing_metadata_checked_at":"2026-09-18T15:41:21.447320079Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.45019569Z","region_metadata_checked_at":"2026-09-18T15:41:21.45019569Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"80bbad9bae888fa5fe44e9dd4cbac874e35d0034b1f9d3ce998648dbbb165cb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","occurred_at":"2026-09-18T02:55:52Z","failed_at":"e2e","signature_id":"a093a69ee54b8dd613584cc9b286724efd055bbbaf92d5a169de74315234601e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce3a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ql74qfp26s59\" to finish creating: GET https://rp.j7111040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\",\n \"name\": \"0964f917-6834-4736-8825-79c12267a054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:34.744618724Z\",\n \"endTime\": \"2026-09-18T04:17:38.586077059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce3a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ql74qfp26s59\" to finish creating: GET https://rp.j7111040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0964f917-6834-4736-8825-79c12267a054\",\n \"name\": \"0964f917-6834-4736-8825-79c12267a054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:34.744618724Z\",\n \"endTime\": \"2026-09-18T04:17:38.586077059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780717720997888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:55:28Z","started_at":"2026-09-18T02:55:28Z","completed_at":"2026-09-18T02:55:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.963572155Z","timing_metadata_checked_at":"2026-09-18T15:40:46.963572155Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:46.966884667Z","region_metadata_checked_at":"2026-09-19T03:30:37.884099306Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"98c29963529b7a2a65bb65ecf53acba8e0780b6c1f009068e660ac6a56ba0bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780717720997888","occurred_at":"2026-09-18T02:55:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:51:54Z","started_at":"2026-09-18T02:51:54Z","completed_at":"2026-09-18T04:53:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.49848179Z","timing_metadata_checked_at":"2026-09-18T15:41:33.49848179Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.502061703Z","region_metadata_checked_at":"2026-09-18T15:41:33.502061703Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"bd14580021d6f01bd288a12840fad362aa2a101364cb346e491f26cacec49252","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","occurred_at":"2026-09-18T02:51:54Z","failed_at":"alert","signature_id":"04a95f9ed56a68aa95bb9d5ccdef76442ca4969785b15ebf86b48f429dfe945c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:04:20Z\n Ended: 2026-09-18T04:18:20Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5911552-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5911552-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100777262654689280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6698,"pr_state":"closed","pr_sha":"40f10113bb5be6fc421531f63692eacc05043a36","final_merged_sha":"75aaa2236a74579c6206715ca27fd0ca55be2ea6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T02:41:44Z","started_at":"2026-09-18T02:41:44Z","completed_at":"2026-09-18T04:36:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:19.520186622Z","timing_metadata_checked_at":"2026-09-18T15:40:19.520186622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:19.52538674Z","region_metadata_checked_at":"2026-09-18T15:40:19.52538674Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100766002009608192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T01:56:59Z","started_at":"2026-09-18T01:56:59Z","completed_at":"2026-09-18T03:53:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.290367689Z","timing_metadata_checked_at":"2026-09-18T15:41:20.290367689Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.2935147Z","region_metadata_checked_at":"2026-09-18T15:41:20.2935147Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100765928928055296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T01:56:42Z","started_at":"2026-09-18T01:56:42Z","completed_at":"2026-09-18T04:03:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.38469403Z","timing_metadata_checked_at":"2026-09-18T15:40:59.38469403Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.387864841Z","region_metadata_checked_at":"2026-09-18T15:40:59.387864841Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100752869291462656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"45f2172a532769a42d1f99fa0ee222c7efa60b40","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-18T01:04:48Z","started_at":"2026-09-18T01:04:48Z","completed_at":"2026-09-18T03:06:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.800209557Z","timing_metadata_checked_at":"2026-09-18T15:41:21.800209557Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.803523669Z","region_metadata_checked_at":"2026-09-18T15:41:21.803523669Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c0ebd76b50205ad901e577ade460296f93b4079e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-18T01:00:47Z","started_at":"2026-09-18T01:00:47Z","completed_at":"2026-09-18T01:30:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.808230852Z","timing_metadata_checked_at":"2026-09-18T15:41:01.808230852Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.811720164Z","region_metadata_checked_at":"2026-09-18T15:41:01.811720164Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"02f23c3c4c01cfeaf268705a12ee578ac8a70edb10cd5df9cb504cbc1f11714c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","occurred_at":"2026-09-18T01:00:47Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-18T01:22:54.255Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T01:22:54.877Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-18T01:22:55.881Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9648512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28\ntime=2026-09-18T01:24:56.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28/operationStatuses/08584119123100595444\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-zhmgezzmxfkbc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j9648512-mgmt-2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T00:42:37Z","started_at":"2026-09-18T00:42:37Z","completed_at":"2026-09-18T02:44:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.359860891Z","timing_metadata_checked_at":"2026-09-18T15:41:33.359860891Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.363504704Z","region_metadata_checked_at":"2026-09-18T15:41:33.363504704Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c1f979c12eb4c64829fb48d994c36008ea43d5333269f88554f4b97ee2c2bc05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","occurred_at":"2026-09-18T00:42:37Z","failed_at":"e2e","signature_id":"82fd789c0354677fc94a578c2ebbcf48d32a60e2464d95dc9709afd5a7216752","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100746534860623872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T00:39:38Z","started_at":"2026-09-18T00:39:38Z","completed_at":"2026-09-18T00:40:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.228241816Z","timing_metadata_checked_at":"2026-09-18T15:41:33.228241816Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:33.231744129Z","region_metadata_checked_at":"2026-09-19T03:30:50.659247726Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f0a4d2a78b6cce5a9956bfb07b3e14edeb5fc1d343d685d099f0eeff8eb2794c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100746534860623872","occurred_at":"2026-09-18T00:39:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]}]}