{"meta":{"start_date":"2026-10-25","end_date":"2026-10-25","start_at":"2026-10-25T00:00:00Z","end_at":"2026-10-26T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T02:57:52Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":17,"failed_runs":14,"raw_occurrences":46,"matched_occurrences":45,"runs_affected":13},"rows":[{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":6,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,6],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-14T12:43:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-q5hjsn\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x18f75457b1e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-q5hjsn: failed waiting for cluster=\"cluster-zstream-4-20-q5hjsn\" in resourcegroup=\"rg-zstream-upgrade-4-20-jb8s5lbcxszk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n \"name\": \"5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:30.073248883Z\",\n \"endTime\": \"2026-10-25T11:30:35.871032188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.42 is Partial (want Completed), started 23m47s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n correlation ID: cf35d619-4e23-43fc-8aeb-ee92df9074e6\n client request ID: 68e2030f-31dd-4592-871e-2e8df326a35b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-q5hjsn\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x18f75457b1e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-q5hjsn: failed waiting for cluster=\"cluster-zstream-4-20-q5hjsn\" in resourcegroup=\"rg-zstream-upgrade-4-20-jb8s5lbcxszk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n \"name\": \"5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:30.073248883Z\",\n \"endTime\": \"2026-10-25T11:30:35.871032188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.42 is Partial (want Completed), started 23m47s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n correlation ID: cf35d619-4e23-43fc-8aeb-ee92df9074e6\n client request ID: 68e2030f-31dd-4592-871e-2e8df326a35b\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x134cca4de620\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-smqlw28gpmfk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n \"name\": \"d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:59.115997025Z\",\n \"endTime\": \"2026-10-25T11:31:07.16366343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m52s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n correlation ID: 9f778ec9-bb64-4e1c-9a2f-a32317e4bb4a\n client request ID: bfadd86d-0c2c-4751-a28f-5658e7a124e6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x134cca4de620\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-smqlw28gpmfk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n \"name\": \"d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:59.115997025Z\",\n \"endTime\": \"2026-10-25T11:31:07.16366343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m52s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n correlation ID: 9f778ec9-bb64-4e1c-9a2f-a32317e4bb4a\n client request ID: bfadd86d-0c2c-4751-a28f-5658e7a124e6\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b59459fc100\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-t7spx49mgg2h\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n \"name\": \"fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:26.130044072Z\",\n \"endTime\": \"2026-10-25T11:30:36.059795656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 22m37s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n correlation ID: 1127ed99-2d51-4906-b522-5ba778a6c853\n client request ID: eb7f7962-a820-4a0a-acaa-426a2d0c04e3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b59459fc100\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-t7spx49mgg2h\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n \"name\": \"fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:26.130044072Z\",\n \"endTime\": \"2026-10-25T11:30:36.059795656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 22m37s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n correlation ID: 1127ed99-2d51-4906-b522-5ba778a6c853\n client request ID: eb7f7962-a820-4a0a-acaa-426a2d0c04e3\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"23e2eb050a4416384db3752c651a9d3db7b8c538cc0bf856ae3091acdd9ca6d1","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"6876faa6b24b00dfa66f80f783d33f416f7acda715020a397c52cb5732457de1","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"8ad851bf221457dbbdf65329df79357aee7c2ceca689a609b15097ec5d216d30","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"c2246a6f55fb01eedd384fdcab8c1f79fabf97dc38129537b05dd14067f93470","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"d9a5d39edfcea9c2671c10762371677ac7311bb7caba7fdb15a801d02fd90b80","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"f763f825d92522261c47dffbfc1adbb9088da2a539dfe81ca7c6af2d8a1d41f4","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":23.529411764705884,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,2,6,3,2,4],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":17,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":4}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-25T18:47:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-bfwrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"97b76fc8-3f29-44d9-83c4-855c4cca2cf8\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-bfwrz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-25T18:47:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3671936-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.215:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-mftk6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c4f6f9ce-1370-407b-a328-72082e9577a0\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-mftk6 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-25T18:50:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3671936-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.215:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-mftk6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c4f6f9ce-1370-407b-a328-72082e9577a0\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-mftk6 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-25T18:50:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-bfwrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"97b76fc8-3f29-44d9-83c4-855c4cca2cf8\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-bfwrz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-25T13:52:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8072320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8072320/providers/microsoft.monitor/accounts/services-j8072320\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-jdq5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9c6d2039-7988-4be2-8890-d290ed7d2d20\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-jdq5f (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-25T13:54:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8072320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8072320/providers/microsoft.monitor/accounts/services-j8072320\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-jdq5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9c6d2039-7988-4be2-8890-d290ed7d2d20\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-jdq5f (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-25T13:55:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8072320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.141:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8072320/providers/microsoft.monitor/accounts/services-j8072320\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-h6zb9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8720bf8b-027c-410b-9181-eafea5b14401\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-h6zb9 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-25T13:55:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8072320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.141:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8072320/providers/microsoft.monitor/accounts/services-j8072320\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-h6zb9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8720bf8b-027c-410b-9181-eafea5b14401\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-h6zb9 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T11:31:30Z\n Ended: 2026-10-25T11:58:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-czwqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"76e86419-4829-44c0-a136-817aadd188d4\"\n Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-czwqq (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-25T11:31:30Z\n Ended: 2026-10-25T11:58:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-czwqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"76e86419-4829-44c0-a136-817aadd188d4\"\n Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-czwqq (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Resolved\n Started: 2026-10-25T11:37:32Z\n Ended: 2026-10-25T11:48:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"governance-policy-propagator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"grc-policy-propagator-5fc7fb4677-67tll\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e5593f36-ff37-4a56-9501-050ee9a5f489\"\n Description: Pod multicluster-engine/grc-policy-propagator-5fc7fb4677-67tll (governance-policy-propagator) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Resolved\n Started: 2026-10-25T11:37:32Z\n Ended: 2026-10-25T11:48:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"governance-policy-propagator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"grc-policy-propagator-5fc7fb4677-67tll\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e5593f36-ff37-4a56-9501-050ee9a5f489\"\n Description: Pod multicluster-engine/grc-policy-propagator-5fc7fb4677-67tll (governance-policy-propagator) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n State: Resolved\n Started: 2026-10-25T11:39:32Z\n Ended: 2026-10-25T11:48:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"governance-policy-propagator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"grc-policy-propagator-5fc7fb4677-j5hzx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"eabaed55-b955-4173-b78d-fec0366c15f2\"\n Description: Pod multicluster-engine/grc-policy-propagator-5fc7fb4677-j5hzx (governance-policy-propagator) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n State: Resolved\n Started: 2026-10-25T11:39:32Z\n Ended: 2026-10-25T11:48:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"governance-policy-propagator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"grc-policy-propagator-5fc7fb4677-j5hzx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"eabaed55-b955-4173-b78d-fec0366c15f2\"\n Description: Pod multicluster-engine/grc-policy-propagator-5fc7fb4677-j5hzx (governance-policy-propagator) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n State: Resolved\n Started: 2026-10-25T11:41:31Z\n Ended: 2026-10-25T11:52:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"ocm-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-pj8f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3ea24fa-145a-42ea-812b-1fe46f10bfd1\"\n Description: Pod multicluster-engine/ocm-controller-859484fc49-pj8f8 (ocm-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n State: Resolved\n Started: 2026-10-25T11:41:31Z\n Ended: 2026-10-25T11:52:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"ocm-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/services-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-pj8f8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3ea24fa-145a-42ea-812b-1fe46f10bfd1\"\n Description: Pod multicluster-engine/ocm-controller-859484fc49-pj8f8 (ocm-controller) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"23a900339c164e91dcce00d9aeb9286d19def65ff9ea3201cee586844076595c","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333714598072320","occurred_at":"2026-10-25T12:30:14Z","signature_id":"10b7f95e300ee35e6c9a51a0b80079fc89d7716cc4d6f052cfff2bc326ebbb9f","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"0a897df4c8f15692a53a241d7a951370aa0b2ed5d1e9e6678c0fa066f18fee55","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"83f9f91a80bdac318fe991b6d8496f6250de9707233dd99d4cad0f40c175e591","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":23.529411764705884,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","occurrences":3},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-25T10:27:22.014Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-25T10:27:22.644Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-25T10:27:22.644Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:27:22.918Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-25T10:27:23.279Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-25T10:27:23.279Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:27:23.445Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:32:28.984Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-25T10:32:28.985Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-25T08:14:44.299Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-25T08:14:45.051Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-25T08:14:45.051Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:14:45.565Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-25T08:14:45.984Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-25T08:14:45.984Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:14:46.156Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:19:54.254Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-25T08:19:54.254Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-25T06:58:33.435Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-25T06:58:33.964Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-25T06:58:33.964Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T06:58:34.148Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-25T06:58:34.349Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-25T06:58:34.350Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T06:58:34.404Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T07:03:38.507Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-25T07:03:38.507Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114287994985779200","occurred_at":"2026-10-25T09:28:34Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114257867568582656","occurred_at":"2026-10-25T07:28:51Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114242927562592256","occurred_at":"2026-10-25T06:29:29Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114223810998702080","occurred_at":"2026-10-25T05:13:31Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.764705882352942,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":41,"prior_last_seen_at":"2026-10-16T07:17:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xd75a95e43e0\u003e: \n DNS for route host agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd75a8fba020\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xd75a95e43e0\u003e: \n DNS for route host agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd75a8fba020\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qx6cd.apps.aro.s4p6p0t9k4o2s8h.fx2j.j3671936.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc4f5ed30b40\u003e: \n DNS for route host agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc4f5f2da4a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc4f5ed30b40\u003e: \n DNS for route host agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc4f5f2da4a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jfjrs.apps.aro.a7b4y7y9a8k0c1j.bzjc.j0985984.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"903e4210ba2623c82b6cf4e8df84550127761ed552ffd7f0056a59a23fd671cb","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"105d71756967099a5d9e4fd8392e89856fa8c1773b28ba775bdc10aba128134f","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.764705882352942,"anchor_occurrences":2,"after_last_push_count":20,"trend_counts":[5,8,0,2,3,0,2],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":61,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T13:57:52Z\n Ended: 2026-10-25T14:23:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b19b2a4f-abec-4efb-8fa1-a0f49de6f865\"\n Description: Pod ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-25T13:57:52Z\n Ended: 2026-10-25T14:23:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b19b2a4f-abec-4efb-8fa1-a0f49de6f865\"\n Description: Pod ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T11:41:40Z\n Ended: 2026-10-25T11:51:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/hcps-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlp722bk3stbjb74e8i42b30lftatrs-ingress-cert-le\", pod=\"capi-provider-6d7bfdf44-24wm6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5602ef00-22ff-45dd-b3cb-39a5ba53e966\"\n Description: Pod ocm-arohcpci01-2tlp722bk3stbjb74e8i42b30lftatrs-ingress-cert-le/capi-provider-6d7bfdf44-24wm6 (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-25T11:41:40Z\n Ended: 2026-10-25T11:51:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/hcps-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlp722bk3stbjb74e8i42b30lftatrs-ingress-cert-le\", pod=\"capi-provider-6d7bfdf44-24wm6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5602ef00-22ff-45dd-b3cb-39a5ba53e966\"\n Description: Pod ocm-arohcpci01-2tlp722bk3stbjb74e8i42b30lftatrs-ingress-cert-le/capi-provider-6d7bfdf44-24wm6 (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Resolved\n Started: 2026-10-25T11:43:40Z\n Ended: 2026-10-25T11:51:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.204:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7524864/providers/microsoft.monitor/accounts/hcps-j7524864\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlp733dhbjgega0lnvcjvgg1grv9tf4-i6w1d6x7y2t6h2v\", pod=\"capi-provider-8f7b799d8-49zjj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"706780f8-3192-4947-b780-05f86e3d1b53\"\n Description: Pod ocm-arohcpci01-2tlp733dhbjgega0lnvcjvgg1grv9tf4-i6w1d6x7y2t6h2v/capi-provider-8f7b799d8-49zjj (manager) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"ec01e424b2be613766af87aa7ffd1f1f917c01aba43f8485f49377c09097c201","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"3fc3c66c8b9815981c4e1256d8181a4ecc52a448d74e0588729208f7e0320945","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1f45b2fce5a9656a648589df07358b5c8f3221613247fa4e2a6dfc2fdd8f6c58","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f211b784120\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fqjfphzpbt4x\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n \"name\": \"dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:22.163428437Z\",\n \"endTime\": \"2026-10-25T11:30:25.784218117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 19m44s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n correlation ID: 16b92b93-ef61-4f4d-9bbb-a63f0c7ebf4f\n client request ID: cc5cc315-91ce-4032-81e3-c0e49c80e860\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f211b784120\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fqjfphzpbt4x\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n \"name\": \"dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:22.163428437Z\",\n \"endTime\": \"2026-10-25T11:30:25.784218117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 19m44s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n correlation ID: 16b92b93-ef61-4f4d-9bbb-a63f0c7ebf4f\n client request ID: cc5cc315-91ce-4032-81e3-c0e49c80e860\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x299d616e8140\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bcvsvbp5d8t7\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n \"name\": \"f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:32.174992685Z\",\n \"endTime\": \"2026-10-25T11:30:36.037388523Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m56s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n correlation ID: 88e2d1ef-c0a9-47ed-aaa6-0693add21a52\n client request ID: 8ab0d68c-218c-40c0-b734-d3980ba8ecab\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x299d616e8140\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bcvsvbp5d8t7\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n \"name\": \"f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:32.174992685Z\",\n \"endTime\": \"2026-10-25T11:30:36.037388523Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m56s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n correlation ID: 88e2d1ef-c0a9-47ed-aaa6-0693add21a52\n client request ID: 8ab0d68c-218c-40c0-b734-d3980ba8ecab\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"782e2be5be125f055ff2ce43c736756398d39d6effe0fee98a9961842d3d84c5","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"fb0f5c839574ca03941c2ecf3ef4dfd30b5ba58990ab84508676591669d4a42b","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T23:59:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x381116f3a0c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n \"name\": \"894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:28:30.111139221Z\",\n \"endTime\": \"2026-10-25T11:52:32.090609688Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n correlation ID: d229c55a-e375-4f40-bb2a-f3ad97610736\n client request ID: aa4bc9f4-d5d8-494d-94b7-252189caf6ce\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x381116f3a0c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n \"name\": \"894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:28:30.111139221Z\",\n \"endTime\": \"2026-10-25T11:52:32.090609688Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n correlation ID: d229c55a-e375-4f40-bb2a-f3ad97610736\n client request ID: aa4bc9f4-d5d8-494d-94b7-252189caf6ce\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x22bced7141e0\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n \"name\": \"954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:21:43.829823922Z\",\n \"endTime\": \"2026-10-25T11:45:52.860717396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n correlation ID: dc5c263c-66f3-4b74-a013-b67fbdf80f9a\n client request ID: 3eb7aaab-ba2a-414a-b973-c3e5012b0c78\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x22bced7141e0\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n \"name\": \"954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:21:43.829823922Z\",\n \"endTime\": \"2026-10-25T11:45:52.860717396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n correlation ID: dc5c263c-66f3-4b74-a013-b67fbdf80f9a\n client request ID: 3eb7aaab-ba2a-414a-b973-c3e5012b0c78\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"080e3030be978a80e1095312a2869c7ed70474ec22ba1b8a90b8a510e75ff0d5","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"aa0717a28ea1274c87a79add45a1c3bfe3bb3254406149bd155ff10e105ee7d1","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df518282228b252a66b7fa05de10041466f90edc2b99ca2d14dd0565ca45d5ae","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zgspqd\" with version 4.22.18 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1120db708700\u003e: \n failed to create HCP cluster cluster-zstream-4-22-zgspqd: failed waiting for cluster=\"cluster-zstream-4-22-zgspqd\" in resourcegroup=\"rg-zstream-upgrade-4-22-4qv6w5xvr977\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\",\n \"name\": \"ffe95342-e02c-4496-883e-5b1bf611d99e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:19.965849634Z\",\n \"endTime\": \"2026-10-25T11:30:25.598823992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, olm-operator, capi-provider, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, cluster-storage-operator, catalog-operator, openshift-controller-manager, cluster-version-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n correlation ID: e700165f-ec9f-48e4-abf2-18b14bbd9d95\n client request ID: cdbafbf0-d5e0-4ed7-8b28-b50e4bf8f6b8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zgspqd\" with version 4.22.18 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1120db708700\u003e: \n failed to create HCP cluster cluster-zstream-4-22-zgspqd: failed waiting for cluster=\"cluster-zstream-4-22-zgspqd\" in resourcegroup=\"rg-zstream-upgrade-4-22-4qv6w5xvr977\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\",\n \"name\": \"ffe95342-e02c-4496-883e-5b1bf611d99e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:19.965849634Z\",\n \"endTime\": \"2026-10-25T11:30:25.598823992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, olm-operator, capi-provider, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, cluster-storage-operator, catalog-operator, openshift-controller-manager, cluster-version-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n correlation ID: e700165f-ec9f-48e4-abf2-18b14bbd9d95\n client request ID: cdbafbf0-d5e0-4ed7-8b28-b50e4bf8f6b8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"6007f17af9a420d188821078ddfcd13ef9adc03b75055acbd2a1cc8c5fb10108","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e1308aaf3b1d1878569e7faaa357169a3760ff01fb807dd5a0044acf0491d40b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x105dbf764180\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mv5sq4l7jv4r\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n \"name\": \"35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:21.658859222Z\",\n \"endTime\": \"2026-10-25T11:30:25.924003842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: capi-provider; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n correlation ID: d887bb88-655e-4cf4-84f8-c59ec1d133f4\n client request ID: 066a4e25-cc0e-4d6f-a9c5-8e71913c3b6c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x105dbf764180\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mv5sq4l7jv4r\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n \"name\": \"35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:21.658859222Z\",\n \"endTime\": \"2026-10-25T11:30:25.924003842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: capi-provider; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n correlation ID: d887bb88-655e-4cf4-84f8-c59ec1d133f4\n client request ID: 066a4e25-cc0e-4d6f-a9c5-8e71913c3b6c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"8c3d03bb192574d4a7d4caa279bb2bc24cd1fc9e953bcb7a6725c28dc34a8baa","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"387b7a3bffcaeae5fc4f0b57ac52dafe4294f605cab90681b8f7faa90ebf4b9b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x1afe2fabc320\u003e: \n failed to create HCP cluster ea-list7tqb74: failed waiting for cluster=\"ea-list7tqb74\" in resourcegroup=\"ea-list-8mbx8mstb22c\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\",\n \"name\": \"506cd809-e6bc-45d8-aa3a-45042c374b41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:00.137646561Z\",\n \"endTime\": \"2026-10-25T11:31:06.609068276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, community-operators-catalog, packageserver, cluster-node-tuning-operator, cluster-storage-operator, ingress-operator, dns-operator, catalog-operator, cluster-network-operator, cluster-version-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, cluster-policy-controller, olm-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n correlation ID: a86e7a8c-dce2-49be-8312-2dfeeebedc58\n client request ID: 03f569a8-daf7-4041-b81b-1efe61e6782e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x1afe2fabc320\u003e: \n failed to create HCP cluster ea-list7tqb74: failed waiting for cluster=\"ea-list7tqb74\" in resourcegroup=\"ea-list-8mbx8mstb22c\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\",\n \"name\": \"506cd809-e6bc-45d8-aa3a-45042c374b41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:00.137646561Z\",\n \"endTime\": \"2026-10-25T11:31:06.609068276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, community-operators-catalog, packageserver, cluster-node-tuning-operator, cluster-storage-operator, ingress-operator, dns-operator, catalog-operator, cluster-network-operator, cluster-version-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, cluster-policy-controller, olm-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n correlation ID: a86e7a8c-dce2-49be-8312-2dfeeebedc58\n client request ID: 03f569a8-daf7-4041-b81b-1efe61e6782e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"35b9f2d778245d855045dd3836882934be33878b78f8ec15263d0fa15d2e6a86","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-29T01:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x22535b109360\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bh6sljsqnwrw\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\",\n \"name\": \"ec3175b8-8979-4854-9d37-2e76e8d09536\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:03.316597788Z\",\n \"endTime\": \"2026-10-25T11:31:06.803246105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, ingress-operator, cluster-image-registry-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, cluster-node-tuning-operator, control-plane-pki-operator, azure-cloud-controller-manager, openshift-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, packageserver, certified-operators-catalog, catalog-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n correlation ID: 7d2b87e8-723a-49a9-a4cb-8027a1fb4066\n client request ID: b5b1bd46-74cb-4267-85b1-e1af986392eb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x22535b109360\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bh6sljsqnwrw\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\",\n \"name\": \"ec3175b8-8979-4854-9d37-2e76e8d09536\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:03.316597788Z\",\n \"endTime\": \"2026-10-25T11:31:06.803246105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, ingress-operator, cluster-image-registry-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, cluster-node-tuning-operator, control-plane-pki-operator, azure-cloud-controller-manager, openshift-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, packageserver, certified-operators-catalog, catalog-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n correlation ID: 7d2b87e8-723a-49a9-a4cb-8027a1fb4066\n client request ID: b5b1bd46-74cb-4267-85b1-e1af986392eb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"774ef136f38611a7eefcb40f8fdd5f5fd82c4b8dae09106b8507e7a803347d4e","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":21,"trend_counts":[5,8,0,3,2,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":55,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x209528ec2340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ktmjg5k5mr4p\" to finish creating: GET https://rp.j2101760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\",\n \"name\": \"cd12106f-dc74-4bb2-88d9-552815761564\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T13:33:58.83934802Z\",\n \"endTime\": \"2026-10-25T14:03:01.452063547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2101760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\n correlation ID: a999ca91-1f04-4667-8416-be09ce537e26\n client request ID: e9d482d5-6b2f-4cbd-99d8-1b5bb0124d29\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x209528ec2340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ktmjg5k5mr4p\" to finish creating: GET https://rp.j2101760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\",\n \"name\": \"cd12106f-dc74-4bb2-88d9-552815761564\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T13:33:58.83934802Z\",\n \"endTime\": \"2026-10-25T14:03:01.452063547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2101760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cd12106f-dc74-4bb2-88d9-552815761564\n correlation ID: a999ca91-1f04-4667-8416-be09ce537e26\n client request ID: e9d482d5-6b2f-4cbd-99d8-1b5bb0124d29\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"acc8285070bd989dffae9a81dcf632a7bf3a9db3d381a859414064975ec30cc3","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"eaaeea36f37284592eadff855e2786beb007e4407cc639524af8e920f7d265fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-vfvmsnwsd5sg/cluster-candidate-5-0-lh7h4v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x274288768300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lh7h4v: failed waiting for cluster=\"cluster-candidate-5-0-lh7h4v\" in resourcegroup=\"rg-candidate-5-0-vfvmsnwsd5sg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n \"name\": \"f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:22.79701272Z\",\n \"endTime\": \"2026-10-25T11:30:25.820223913Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 20m56s ago; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n correlation ID: d572d016-0277-4113-8a96-1da300eb89da\n client request ID: a981c6e6-813e-4373-81f7-4aab2cf9a886\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-vfvmsnwsd5sg/cluster-candidate-5-0-lh7h4v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x274288768300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lh7h4v: failed waiting for cluster=\"cluster-candidate-5-0-lh7h4v\" in resourcegroup=\"rg-candidate-5-0-vfvmsnwsd5sg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n \"name\": \"f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:22.79701272Z\",\n \"endTime\": \"2026-10-25T11:30:25.820223913Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 20m56s ago; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n correlation ID: d572d016-0277-4113-8a96-1da300eb89da\n client request ID: a981c6e6-813e-4373-81f7-4aab2cf9a886\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"dff12b42279b00707352b18af2f0050316e2032026e4d08cb488796c14d37bc7","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6b9eb312202879fb9a6f390df391c95e64889c51047a74b640051be8935c181f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-29T01:37:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a8a07a1e540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rvs4gzxj9lgg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\",\n \"name\": \"63403578-fb62-456b-b40e-41ef79e2e399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:39.014043295Z\",\n \"endTime\": \"2026-10-25T11:31:45.790547256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m13s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n correlation ID: 38478f64-7ba7-40aa-b720-a7f94d356d8b\n client request ID: 82836f4c-341e-4d1e-88f4-c55b4e681a30\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a8a07a1e540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rvs4gzxj9lgg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\",\n \"name\": \"63403578-fb62-456b-b40e-41ef79e2e399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:02:39.014043295Z\",\n \"endTime\": \"2026-10-25T11:31:45.790547256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m13s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n correlation ID: 38478f64-7ba7-40aa-b720-a7f94d356d8b\n client request ID: 82836f4c-341e-4d1e-88f4-c55b4e681a30\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"dcd430427d782eeea86e74cee94b16d2a3435cb845ddf1163daad51281789a78","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"663e8d8eeb1c1b6a6d00064673e9a3330c994712d05117a165f749566b26a621","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a59ed8de2a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-xf7h6gdhg87b\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n \"name\": \"0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:20.835629052Z\",\n \"endTime\": \"2026-10-25T11:30:25.907970891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n correlation ID: f5a00756-4d2f-4428-b3ea-e6cf5e3ef0a1\n client request ID: 03254b25-a5c1-4e3b-88dd-5aeb8bec557c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a59ed8de2a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-xf7h6gdhg87b\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n \"name\": \"0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:01:20.835629052Z\",\n \"endTime\": \"2026-10-25T11:30:25.907970891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n correlation ID: f5a00756-4d2f-4428-b3ea-e6cf5e3ef0a1\n client request ID: 03254b25-a5c1-4e3b-88dd-5aeb8bec557c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"5d52aeb2aaff4eeba7d8d90ff684f12b2a5726d50ed448947775022cdfac17e6","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,1,0,1,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-16T05:08:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a5fafb1fdc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dhxpqc2757nk\" to finish creating: GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n \"name\": \"995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:09:27.17231924Z\",\n \"endTime\": \"2026-10-25T11:28:33.403436193Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n correlation ID: d01faed1-7c94-4749-8892-78ade8753a54\n client request ID: 5d2be0a3-7126-4924-b3ac-5c1c229ea51d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a5fafb1fdc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dhxpqc2757nk\" to finish creating: GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n \"name\": \"995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:09:27.17231924Z\",\n \"endTime\": \"2026-10-25T11:28:33.403436193Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n correlation ID: d01faed1-7c94-4749-8892-78ade8753a54\n client request ID: 5d2be0a3-7126-4924-b3ac-5c1c229ea51d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114291888369438720","occurred_at":"2026-10-25T09:44:02Z","signature_id":"1187688d249cfcb086795f44465ce4133550dfe21fbda93d7619f7c1cd0c8e0e","pr_number":7657,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":14,"trend_counts":[2,6,2,0,5,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-16T13:20:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x34e2e46f8680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mfmrpz5xbrdj\" to finish creating: GET https://rp.j1877504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\",\n \"name\": \"3d1a2520-89cf-4c42-a91c-ec725b89af14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:04:09.039925979Z\",\n \"endTime\": \"2026-10-25T11:23:09.310567948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1877504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\n correlation ID: 9acdbad6-f09a-46a0-a926-c738c62f3c07\n client request ID: 916943ff-0406-46da-9830-2889e6355ff3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x34e2e46f8680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mfmrpz5xbrdj\" to finish creating: GET https://rp.j1877504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\",\n \"name\": \"3d1a2520-89cf-4c42-a91c-ec725b89af14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-25T11:04:09.039925979Z\",\n \"endTime\": \"2026-10-25T11:23:09.310567948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1877504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d1a2520-89cf-4c42-a91c-ec725b89af14\n correlation ID: 9acdbad6-f09a-46a0-a926-c738c62f3c07\n client request ID: 916943ff-0406-46da-9830-2889e6355ff3\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294657641877504","occurred_at":"2026-10-25T09:55:02Z","signature_id":"a8e2229b6df36d2c0aba95f4704bbac9bcfb7b27ecdb484adcdb5f787901f157","pr_number":6081,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-14T19:26:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0x2401cda17700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-6lgfpz\" in resourcegroup=\"rg-np-labels-taints-2pq6dbldqrjk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: f50cf33f-8bc4-4ce0-8a7c-d816cfc4e960\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-bvffjvcy6qo3o' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n correlation ID: 9ef1eaf3-8ba8-425e-8fe5-70f8311af59e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0x2401cda17700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-6lgfpz\" in resourcegroup=\"rg-np-labels-taints-2pq6dbldqrjk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: f50cf33f-8bc4-4ce0-8a7c-d816cfc4e960\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-bvffjvcy6qo3o' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n correlation ID: 9ef1eaf3-8ba8-425e-8fe5-70f8311af59e\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"3d343371079ea4c8eec5fdc9bbbf22e9c2c37580483fe6a2b69023d256138a61","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4a04fe062c510b6c532fad92f851f6d9fcd4a47e168844e52656b2097c3939c","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-rdkvkj' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cprxqg\"\nUnexpected error:\n \u003c*errors.joinError | 0x30c853d502b8\u003e: \n VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-rdkvkj\" not yet present in zone rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-rdkvkj' does not exist in resource group 'rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n correlation ID: 984d9387-98d9-461e-8f5b-657273ddef57\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cprxqg\"\nUnexpected error:\n \u003c*errors.joinError | 0x30c853d502b8\u003e: \n VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-rdkvkj\" not yet present in zone rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: NotFound\n --------------------------------------------------------------------------------\n {\n \"code\": \"NotFound\",\n \"message\": \"The resource record '*.e2e-dns-rdkvkj' does not exist in resource group 'rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n correlation ID: 984d9387-98d9-461e-8f5b-657273ddef57\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114494146650574848","occurred_at":"2026-10-25T23:07:44Z","signature_id":"ecf89ec70a260529697f24e15845fd8acfb2ceebcc45bfb36b26f438351b97bf","pr_number":7739,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":36,"trend_counts":[4,9,1,3,3,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":109,"prior_last_seen_at":"2026-10-15T19:10:04Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T14:09:53Z\n Ended: 2026-10-25T14:23:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xj2t5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-25T14:10:54Z\n Ended: 2026-10-25T14:19:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xj2t5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"c478c5b907afdf2bf8291383bc1b8fdf82880ef962724d36b9c0e9c4f0ffc1c1","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":41,"trend_counts":[13,8,11,4,3,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":155,"prior_last_seen_at":"2026-10-18T21:09:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 34 time(s) (3 unknown, 31 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T11:42:42Z\n Ended: 2026-10-25T11:50:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tlp73656llsbjr851ct21vm7ag358hj-cilium-cl\", pod=\"azure-file-csi-driver-controller-66f6c89876-l6fts\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tlp73656llsbjr851ct21vm7ag358hj-cilium-cl/azure-file-csi-driver-controller-66f6c89876-l6fts has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-25T11:44:40Z\n Ended: 2026-10-25T11:50:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tlp73656llsbjr851ct21vm7ag358hj-cilium-cl\", pod=\"multus-admission-controller-58547c8cf6-lmftl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tlp73656llsbjr851ct21vm7ag358hj-cilium-cl/multus-admission-controller-58547c8cf6-lmftl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-25T11:53:40Z\n Ended: 2026-10-25T11:59:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7524864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tlp70no771drpm6jf2csmj08ibs461h-s2r6f9c0k1q1q9w\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tlp70no771drpm6jf2csmj08ibs461h-s2r6f9c0k1q1q9w/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"c88b83725f4c290056abe9fc249d58727c9801758819fd002dadddfc47f86675","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[5,8,0,2,2,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":61,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T13:55:59Z\n Ended: 2026-10-25T14:19:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xj2t5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-25T14:06:53Z\n Ended: 2026-10-25T14:19:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2101760-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.228:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2101760/providers/microsoft.monitor/accounts/hcps-j2101760\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xj2t5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tlrehr6k1m1drteiq3dbt6soul2kbq3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"3198e55fd62b864e78aeda30d29dbd665bc483504cacc93fd6ae861431463810","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-25T11:11:08Z\n Severity: Sev3\n Labels: alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j1877504-svc\", severity=\"warning\"\n Description: Credential refresher expiration metric is missing for service cluster ci01-j1877504-svc."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294657641877504","occurred_at":"2026-10-25T09:55:02Z","signature_id":"2a2f274dff4f6f6fc65910529a8fbb12b08a54cbcdfc227ca6f705b9a23049d4","pr_number":6081,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6650ca1290fb5ed06584e1395a3de82b930b52fc740efa2bb1408633ebdaa7dd","failure_pattern":"alert [svc] FluentBitIngestionPaused fired","search_query":"alert [svc] FluentBitIngestionPaused fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FluentBitIngestionPaused does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FluentBitIngestionPaused does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T11:48:54Z\n Ended: 2026-10-25T11:58:53Z\n Severity: Sev3\n Labels: alertname=\"FluentBitIngestionPaused\", cluster=\"ci01-j7524864-mgmt-2\", component=\"monitoring-infrastructure\", pod=\"arobit-forwarder-q4jd5\", region=\"westus3\", severity=\"warning\"\n Description: Fluent Bit pod arobit-forwarder-q4jd5 on cluster ci01-j7524864-mgmt-2 has paused collecting new log data for at least 5 minutes.\nIngestion pauses when Fluent Bit\\'s internal memory or storage buffers are full, typically caused by backpressure from a slow or failing output.\nInvestigate the Fluent Bit logs for the specific error details and check the Kusto instance health."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"98720abe90ff53bf328a9b4ffc00853a75870d3a5d84137af7e1ce00be424528","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,1,6,1,1,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-15T19:10:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-25T19:03:55Z\n Ended: 2026-10-25T19:10:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-25T19:03:55Z\n Ended: 2026-10-25T19:09:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-25T19:03:55Z\n Ended: 2026-10-25T19:10:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"8803add155979278d8c0e23a00c75b642794b1490db265041a71fc2a6c5a337f","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[3,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-16T13:42:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x25b4e2d279a0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x25b4e2d279a0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"5ceb3b0e570a6fbb8e99110224b2256342313e80153bb81dc65b77bbad401dd6","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,1,4,1,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1}],"full_error_samples":["time=2026-10-25T11:53:02.786Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-25T11:53:04.190Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-25T11:53:04.190Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8850688-svc\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=3m5s\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.207Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-25T11:56:11.207Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.237Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-25T11:56:11.450Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.510Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j8850688-svc\ntime=2026-10-25T12:11:14.972Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114315309748850688","occurred_at":"2026-10-25T11:17:06Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7597,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7cf505551991c2dd368b5369e4fad74b682af51f37d72a779eab2ae641b8364d","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2f3433d553d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2f3433d553d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"8d9d7ef8eccc46082983c78c6940e3a331458157cfab732b80148db78e3f5fd5","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"73bb7634af35e831eaf0b0bd52966c2235200ca7b8f76813dbe2e28963b542ce","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 620.639s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1cfcad623430\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 620.639s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1cfcad623430\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333714598072320","occurred_at":"2026-10-25T12:30:14Z","signature_id":"090ccca170527876a3cb6a1108982d860d57ba3a86683c697ef9e7b7b0f84311","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40550f1869e8f917cf0ba372be4f16663c844fbaf77e07168049278696500fd1","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-19..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 630.763s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2d6e687aea00\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 630.763s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2d6e687aea00\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"4089890b6b331518bf34a303952645ae56025b4bf66de4696178dbcd0ea811a5","pr_number":7712,"after_last_push_of_merged_pr":false}]}]}]}