{"meta":{"start_date":"2026-10-22","end_date":"2026-10-22","start_at":"2026-10-22T00:00:00Z","end_at":"2026-10-23T00:00:00Z","timezone":"UTC","generated_at":"2026-10-30T03:29:39Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":11,"failed_runs":6,"raw_occurrences":91,"matched_occurrences":89,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":10,"after_last_push_count":23,"trend_counts":[0,0,0,0,9,0,10],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-15T06:27:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00c7da51be8e89ce15ce8e10590e25d85b2282252d307b006276577d947b9e86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"01479cc8260099169681c3c9aac7e1601c47f3919a6c41c57610a9af6d7dfa91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"2ca8d85c3dc2b47b78c6a24945b274f76b13627c62c8a5c01d3ebc927fad0021","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"541341a5ff6a403f156c133d1168366027c4c77d725975906770d24cc82a1c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8443649e9083d1ab2db3cf3952a87eb977678118008a762dcd368311d2aaac93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b311e9b9b11e8b11ec8d0f86f4d29406e7d83165875e0863fb9281515f0c7523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ba521c2c19919a5caa070da355f36879cb81eb48d013d05ff609671940da6a95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"c0bc7ce2f8badc2267d7a65ad1002f4d46160b2c3b4087c1135728e23c2b816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e4e0808ae7571f998df5b39264b674c05cb041138f2cb2532613af052179af56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f7d5f9aaa39bc4f25aca130a942f0670850a3d28be7139dc0a9df1e58b8e9577","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":9,"after_last_push_count":11,"trend_counts":[0,0,0,0,2,0,9],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"105a43b4bf54437760ddaec80d7274c76e0880fdd04198d3317c96b1e41e0747","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"179000d9fd7136a36d03dbd3bfaa3c8edbf92638e8ad95333390e6c1eadcd7dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"19248e56dbbe622fc0ab707175608011b12ef937b74bee06e900da1ff17799d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7532d5417c94c768b19029dc4b5188f6d4f7b0525e268ed507fdd211a7691731","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"78bb0fd5dc8b2957b03be693a8b15a24b0624bf8afc1be3e13738a6518bfff22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"89edbfd547763da1be2ecb5ac3a37c254a734ffe88e2c8157270658bdc6cd116","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cb6c62fc5bd1672b019609253e5970cf0be960ae29f59adef3f3e3018a703317","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e50814639bbed4fd159acc63cc76eb3b3ea44eedc3c0972c63ffedfeeb1d6b35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fa2dafb46bcd964c71b90a558a536dd7120ddbfa5212144072e39dc9ca2ed0c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":9,"after_last_push_count":12,"trend_counts":[0,0,0,0,2,0,9],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-12T13:01:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"012587d7d4630c81e42093fd6ab2c1257951217e2525d7784fd89a2e89112676","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"1e540a6273c70580b8d0a813a5ac2b3184c764a6be285464f22624d92e6da8b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3869c78574390577fcc299556aef741fbec4d0cca2bc649c0c2a2e1cd5f36f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3dd0847594fe9934ecaef404cfd5b2e0d30bdd83e3bdd0629d37df342bb8fe04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7a1cec7cd36f34b86775ec4e998ac79d4f3902ec309e0df7dd447a697e1ef501","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"884359bc37b7b0f624f652c6e3ab6489f45271d6a5f2eaf45cf5d681cd51b93c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"940d1146406ccc6a3297d550ec44a0b5d1a43287589036549f13183fd87e514e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d15e0ee9a765947a51994971a21dbf7f7f640ff3a8e13f8d501e97316301c1e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f052c83930df14da60956d205c94d7b6903bdbd091c343818e9c8edabac6c7c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"06ff6d227a4a567e0691243be5eccb0caeeaaa315bcc42dc0fea399f15c10736","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"2cc2591b74b05456e73089bc9b36c0f1f7eee60669fc135f3ce15bfec52d2937","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"4fa8b7fc464071338bdd4561210059a3e2c75c44318bf3c18a41e27d54116b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6f30c7a8231594aeb46f3d54216c8a117b477376ff72a9a27924ceba4f803e6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"aca4445f1d5148a20ae5983cba5de62c36b6c0a7b69db5f707a5e755bb5182a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"dd0b0b8df7b2166bc054ede8f1d58b688dc0dac527d6b2fd66ecb6ab9cb8ddc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fe77c534f2d11553fbb56bdbdd25fa10e6b63e20abc56954df520c44f8dde521","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":7,"after_last_push_count":19,"trend_counts":[0,0,0,0,12,0,7],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00e889108d18b2602a9701e81c628e0117c6af2ac6506f48e3dc6377714f2dec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3b522484ec379b7598bbd268e9f04bacd0edc777033a068970a7c75bb7fd3ebc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"51ee050adec069064368b692e2e9f92789ea23eb4ea27242862d06944731e6a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"93b4242fbca78a6804343fa0c780a64cee3b6f7f909e539aa95b885d3208f48b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a705768924b8c91914dfee887fd041226da8b0d2a7c49c71af1ef216422bf744","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"af774aac99c529ad98d882b83b9a1e38d054f96b4ba76e020f5318718f9b0f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d28602bc01604b20a4fd1810051138a3b4d9daa151d24a925a551647d550c3c4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"022548c08c7f3d00dcc2c6ccfbb90c7621484b0f6cc92b83282d239b2bf97fec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"0705e7ce0972714a7b11f80b2c8aff85f61cdc6febe2637f5a146fe039b1f537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"3fb0e1297078af590b2d19465e2eb4a0832651168d99f0176c33e483a935dcbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6d820660123e1f2b46bada2acc37a3d7025031ef13070c0759df5cdf0c7b2de4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"b8acb5f0bafdb5a3735bb27828a6b4f3c5003359238914449eaa893ab955598b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"e4a1953e3872ab73515a0438ce007de9e2def00ae922c3087992e0fce1014e4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fd25ada7f7c17331da353bb415e7abc6bfe24733b76e6dae5911dba3aa007638","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":5,"after_last_push_count":11,"trend_counts":[1,2,2,1,0,0,5],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-18T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x14719e7fa150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x14719e7fa150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe4d0c558c40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe4d0c558c40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x37eb40681150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x37eb40681150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"28f82d6598d8560878485b881cedade2766f6f99a7665f52a5642ec9863e8f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"554520898e3bf65aa33d08c67763b5d08098c2dea88239fbc79acfa28b30250c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"91144e10f499ef01ce91bc27b79924e73fc51fedcf3381a2242dc3972585a568","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"ad2974ff484501a74259c346bde4f86a21cbaef053d5d94585993a7186adc875","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"d6194b621062ac0dc7417d99a6662052b73f0727482224784de827ab3d686809","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-zn6lkv\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1288c746e940\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x1288c746e900\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x1288c746e8c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x1288c8648ef0\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-zn6lkv\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1288c746e940\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x1288c746e900\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x1288c746e8c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x1288c8648ef0\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9frg2v\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26bc41040e40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26bc41040dc0\u003e{\n            observation: \u003c*errors.errorString | 0x26bc40ab97d0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26bc41040d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9frg2v\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26bc41040e40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26bc41040dc0\u003e{\n            observation: \u003c*errors.errorString | 0x26bc40ab97d0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26bc41040d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-w29x8s\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2369de188ea0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2369de188e60\u003e{\n            observation: \u003c*errors.errorString | 0x2369e043f830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2369de188d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-w29x8s\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2369de188ea0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2369de188e60\u003e{\n            observation: \u003c*errors.errorString | 0x2369e043f830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2369de188d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"11a1ea4698faced47725bf493b4aacc2124922e05d2aed8e74789e91de7a5717","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"60d8736178a39e3a5b4fef671fa2ef1330288eda4204ffbc60e31862e2471d95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"7d7a4bc924021f4fa04d801c41a3ac27180abe1576c70ed76bd677cebe7f2a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"a0af21048f0851309014ba5e3dc0d0f72ace62a5a58aa35d7e2b48b85ae30c13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"e51db32a50ceffa0ea6a335e06c32c375c606cf3ce2f73a2c6b002fa3305aea2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[0,0,0,0,3,0,4],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"30b3256251eebbc6ad2f2d21745e8e3e6c242862e551efcc5662d3dab722db4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"37af00141384f00be88a064e4ee9337863ec4649e69343bc6f3df05230fa74a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"610533cebef7feda8d458dc8e967f9408265cfa9d327e5638860a65b8ee27823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b542846af6df57de6bbea4872c890fa54411aeb1201b4a017c6a8bfb54ce2c21","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[0,0,0,0,4,0,3],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"0e6ceee3cd3e93558b27b65ed689870c63b26dd3bc15ef1fe3541ac45579879e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"6d20fac6b2c664e06dd79b23d104b0707d6bcfcfba1dad3f3a5feeea9f41e97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"74f59f951658f16f8d54e8163c072eeb6209baea5d1369c72844e0d3da80a25d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a6864331b74f9520caaa01fb142d42bd934a1b337ffea6fcc51fd907a468b304","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d3dedfa37fc8f8f9d48c596be2117ea1ea85b98d4f8b0aba0451aa1f7bea81bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fd02807286b5b83c10835eef9428f86b7fe1936bd74115749fc73fc5111727ba","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"8a9ded4e992ead494e672f2c33210492690aab0b14e2de539e32782deff72275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cbeccff849e6332c34aba59d393fdbae4f3ec1001f41174ac4f54440f6912fdd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,0,3,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"10bb0bdb748cd2f51cfdd05065854a330965cf2734479264377206588e187b5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8181e1d728fd2f9d65a27b1a6f385b120f5a9c516932134b128d158f80fe37e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,0,0,6,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96c75f1ee40922482fa619e484710e1e93eb36cf6eaf5f78027744f7acdd09b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d2d766d5aec5f079fb45688d592a83dedffde25333099326b9f7f5f8e100d8f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"43ca0ef5137db4c92687ca44af2e5269c14f5a14f38084e6f18c59c868e18577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"66fe9486ebbdab2264fdca019d93b069038bbe27a27a787cb2a4da95a08e65ec","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3ec3f52024a46a1f942ae3af19740a4993c313ed0bf4cec5b820ab716098ba7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96e526637370066fb982529359f8fb2c1357e11506ca6d9024d12db800540150","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,4,0,2],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"222ffc9c563b0b4467dc853c34c77c4694cb2ce41c4c51c01cfbc0965f89f195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3918b5053e2c1b6926508abb7cd5f09e8c960484f7c759d88c7b0673e703991d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ac79520c264a03cdae111d50487f33410c268ee3536a309598ebaf8fa5495141","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"5a8c81cfc5de57e2ac5cb1f344b5d4c790956481a2f66094e81c030894d26e74","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fffa36f85bbbff9b9e35ac04e5e1995d6605346e9b86d9872df10486fda0e14f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a7827d5ad5ffd2899a170b78f82132d6020ab7d4fd7644e60ac3a32d23a3e66e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"9d48f25db5125a951413a47642b0efecb6d5d06c908e8b1e5fde16b1b20e1440","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b9ac8e13134319c4cdeffb9d46f43716baeb67c5239b74632e6bf7675546e7af","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"bace6221e58a5d21dfe413ecbb1909743fd07f6b8c34c992d650b4df4119cf4a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"12318b0632fb674fc784fdd096e34ba234440b0cb9593f4ff4c36061f31941fe","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" in cluster=\"\u003ccluster\u003e\" resourcegroup=\"\u003cresource-group\u003e\" to finish updating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6b805215ae5be9a20646349cb83be2fc300cf869ef7d92e96198f6d3b0f5e8fe","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-16..2026-10-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f4c91ed9f592165df1c3c27e2fd22c4b93dc93ae321a35275a1f80ee00060e71","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}