{"meta":{"date":"2026-10-28","timezone":"UTC","generated_at":"2026-10-28T23:55:41Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T19:00:12Z","started_at":"2026-10-28T19:00:12Z","completed_at":"2026-10-28T21:47:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T21:56:37.135525384Z","timing_metadata_checked_at":"2026-10-28T21:56:37.135525384Z","region":"eastus2euap","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T21:56:37.138559775Z","region_metadata_checked_at":"2026-10-28T21:56:37.138559775Z"},"failed_at":["e2e"],"failed_test_count":63,"occurrences":[{"row_id":"0490b5c848afb094d3350927e52fe71082de75ada31620f43e7b2f37e4d00ab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"bec54b9ff9b28304b3228ba00ce3498e04da34d0d0ff7192696df423f2a05e7d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-1-l85jxrxmvfg7/cluster-candidate-5-1-ghbggt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x709092aac80\u003e: \n failed to create HCP cluster cluster-candidate-5-1-ghbggt: failed waiting for cluster=\"cluster-candidate-5-1-ghbggt\" in resourcegroup=\"rg-candidate-5-1-l85jxrxmvfg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\",\n \"name\": \"ac8055d4-30a4-4ff3-95b6-1f02d345f839\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:42.248648198Z\",\n \"endTime\": \"2026-10-28T20:39:45.04100346Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\n correlation ID: 2a70045d-9d54-4fca-878d-5f4bb11971b7\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-1-l85jxrxmvfg7/cluster-candidate-5-1-ghbggt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x709092aac80\u003e: \n failed to create HCP cluster cluster-candidate-5-1-ghbggt: failed waiting for cluster=\"cluster-candidate-5-1-ghbggt\" in resourcegroup=\"rg-candidate-5-1-l85jxrxmvfg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\",\n \"name\": \"ac8055d4-30a4-4ff3-95b6-1f02d345f839\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:42.248648198Z\",\n \"endTime\": \"2026-10-28T20:39:45.04100346Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac8055d4-30a4-4ff3-95b6-1f02d345f839\n correlation ID: 2a70045d-9d54-4fca-878d-5f4bb11971b7\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0dbeee451a53f148651f56d00e59578187aeca8551394204a9dc74d8f42e424e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"25962cc73cb06a1728ac605aa1f4b06859ed88a0569ae6115f498a1cbc7e4539","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-s7k2sd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x297062e2e820\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster=\"cp-ystream-upgrade-4-23-s7k2sd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-np7xjvwc8xct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n \"name\": \"5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:32.825742556Z\",\n \"endTime\": \"2026-10-28T20:41:35.067951058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-s7k2sd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x297062e2e820\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster=\"cp-ystream-upgrade-4-23-s7k2sd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-np7xjvwc8xct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n \"name\": \"5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:32.825742556Z\",\n \"endTime\": \"2026-10-28T20:41:35.067951058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e9c3ff171346124e86b049427a83cb6c0049115f36edeb1ab9b21fce07992dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"19b82bfd691cff869c6c199782be4eee2c38c25cbdf6f1433c85a73266a7223c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3da3f19209c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-p9cvh6b6lj55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n \"name\": \"9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:36.875741431Z\",\n \"endTime\": \"2026-10-28T20:43:44.887394978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, dns-operator, catalog-operator, certified-operators-catalog, csi-snapshot-controller-operator, redhat-marketplace-catalog, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, packageserver, ingress-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n correlation ID: 84abeaa9-4051-418b-b066-32246401955a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3da3f19209c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-p9cvh6b6lj55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n \"name\": \"9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:36.875741431Z\",\n \"endTime\": \"2026-10-28T20:43:44.887394978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, dns-operator, catalog-operator, certified-operators-catalog, csi-snapshot-controller-operator, redhat-marketplace-catalog, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, packageserver, ingress-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n correlation ID: 84abeaa9-4051-418b-b066-32246401955a\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37985da5f97bff5443a20ce7eddabdbfd55a0839dc6b22fe1cac002e46ccec2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-controller-manager, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2245aea75b26f22e3d67aeff6c0c0e5a48771d541a3d789ea360cc23bf8fc67e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ef331e26d1592f4da8d5a5d7392e5167b1dc464a19ba6a9519786efd70bd586d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-jqd7cs\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x13016dc5e5e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-jqd7cs: failed waiting for cluster=\"cp-ystream-upgrade-5-0-jqd7cs\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ft5czs88jtdf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n \"name\": \"1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:34.851456324Z\",\n \"endTime\": \"2026-10-28T20:41:34.873934441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n correlation ID: 03cba8d0-c85c-4970-a348-de9b3f376ff4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-jqd7cs\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x13016dc5e5e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-jqd7cs: failed waiting for cluster=\"cp-ystream-upgrade-5-0-jqd7cs\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ft5czs88jtdf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n \"name\": \"1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:34.851456324Z\",\n \"endTime\": \"2026-10-28T20:41:34.873934441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n correlation ID: 03cba8d0-c85c-4970-a348-de9b3f376ff4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"244ba7cd30b095eb60107a18b189fc64b6784df8154903fa4d1553bd8aabea34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"fd12dbccec9d53cfb9ca9f8ad5c7dd7ccc1665086dac2a9a1dfaa7d925d023d0","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g5tsjc\nUnexpected error:\n \u003c*fmt.wrapError | 0xd7eab0d0620\u003e: \n failed to create HCP cluster np-noedge-g5tsjc: failed waiting for cluster=\"np-noedge-g5tsjc\" in resourcegroup=\"rg-np-noedge-g5tsjc-pf5lpmzmdnbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\",\n \"name\": \"36728157-067b-4375-a959-9c9ce43e534b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:56:26.269135944Z\",\n \"endTime\": \"2026-10-28T21:25:30.687402107Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m57s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n correlation ID: ff24c2ee-7aff-4c46-91cc-8fde998760f5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g5tsjc\nUnexpected error:\n \u003c*fmt.wrapError | 0xd7eab0d0620\u003e: \n failed to create HCP cluster np-noedge-g5tsjc: failed waiting for cluster=\"np-noedge-g5tsjc\" in resourcegroup=\"rg-np-noedge-g5tsjc-pf5lpmzmdnbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\",\n \"name\": \"36728157-067b-4375-a959-9c9ce43e534b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:56:26.269135944Z\",\n \"endTime\": \"2026-10-28T21:25:30.687402107Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m57s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n correlation ID: ff24c2ee-7aff-4c46-91cc-8fde998760f5\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"facbcb79129e6159c9b5455a0c79d2a7fec07d572d42676eaad216f2de41ba83","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25be9887baf7602e2437ace8971617a84c4a0c01503a315daf90b8a9168e180c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"1b112c4de17bdc1ed6df2e4af8db0e4c69a28c7d216afb2f5acd996a62fa50e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-qtj68sbhkffm/cluster-candidate-4-20-hxsx6r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15a8256fc4a0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-hxsx6r: failed waiting for cluster=\"cluster-candidate-4-20-hxsx6r\" in resourcegroup=\"rg-candidate-4-20-qtj68sbhkffm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n \"name\": \"93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:04:15.708912895Z\",\n \"endTime\": \"2026-10-28T20:33:16.216619801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, olm-operator, cluster-storage-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, packageserver, catalog-operator, cluster-policy-controller, cluster-version-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n correlation ID: b596877d-09f6-4478-a504-ebb349eebdd1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-qtj68sbhkffm/cluster-candidate-4-20-hxsx6r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15a8256fc4a0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-hxsx6r: failed waiting for cluster=\"cluster-candidate-4-20-hxsx6r\" in resourcegroup=\"rg-candidate-4-20-qtj68sbhkffm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n \"name\": \"93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:04:15.708912895Z\",\n \"endTime\": \"2026-10-28T20:33:16.216619801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, olm-operator, cluster-storage-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, packageserver, catalog-operator, cluster-policy-controller, cluster-version-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n correlation ID: b596877d-09f6-4478-a504-ebb349eebdd1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25e746e6725143587644e15ad5243ca0a366aaeaa32253462bd988e130f4198e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"0d94360a0b18b44f41203aa87857d79b85bdb58471c8bb507781e2c8936eb1c5","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x34cbd3d68900\u003e: \n failed to create HCP cluster ea-list8sj7pd: failed waiting for cluster=\"ea-list8sj7pd\" in resourcegroup=\"ea-list-qk95gg7gpcnb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n \"name\": \"91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:32.731497539Z\",\n \"endTime\": \"2026-10-28T20:43:34.905565303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-collect-profiles, community-operators-catalog, openshift-apiserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, csi-snapshot-controller-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, kube-scheduler, ingress-operator, cluster-image-registry-operator, packageserver, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n correlation ID: 92e26e96-cd60-4139-8146-f029c76dbd9e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x34cbd3d68900\u003e: \n failed to create HCP cluster ea-list8sj7pd: failed waiting for cluster=\"ea-list8sj7pd\" in resourcegroup=\"ea-list-qk95gg7gpcnb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n \"name\": \"91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:32.731497539Z\",\n \"endTime\": \"2026-10-28T20:43:34.905565303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-collect-profiles, community-operators-catalog, openshift-apiserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, csi-snapshot-controller-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, kube-scheduler, ingress-operator, cluster-image-registry-operator, packageserver, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n correlation ID: 92e26e96-cd60-4139-8146-f029c76dbd9e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"27cb877b508a310c640642bd71151ad6243dce3da617e5ab0f84708a504a18f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"36ab0f864c0e4e4ebea7cdce2d76b37ddec10828ae4d6d3f57c35bc6ef336362","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd149dec0ae0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6nnwct2hkjrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n \"name\": \"89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.936404311Z\",\n \"endTime\": \"2026-10-28T19:53:23.678300779Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-version-operator, openshift-controller-manager, kube-storage-version-migrator, catalog-operator, azure-cloud-controller-manager, dns-operator, redhat-operators-catalog, cluster-storage-operator, control-plane-pki-operator, openshift-route-controller-manager, hosted-cluster-config-operator, packageserver, konnectivity-agent, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, certified-operators-catalog, machine-approver, cluster-network-operator, ingress-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n correlation ID: c78028a7-799f-4fe4-8afb-462431828f6c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd149dec0ae0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6nnwct2hkjrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n \"name\": \"89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.936404311Z\",\n \"endTime\": \"2026-10-28T19:53:23.678300779Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-version-operator, openshift-controller-manager, kube-storage-version-migrator, catalog-operator, azure-cloud-controller-manager, dns-operator, redhat-operators-catalog, cluster-storage-operator, control-plane-pki-operator, openshift-route-controller-manager, hosted-cluster-config-operator, packageserver, konnectivity-agent, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, certified-operators-catalog, machine-approver, cluster-network-operator, ingress-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n correlation ID: c78028a7-799f-4fe4-8afb-462431828f6c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"29f1e6085070ae2b0557bad396daefa56454f148a1307d397d6969b0c8ec6de4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"9c6b7c5bde616122f7d939bcc3b19043ed6c57c2c81f1b37441d253d22839f33","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x2190c42a2260\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nd5rwvspn9b8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\",\n \"name\": \"71207653-6aed-4ccc-a2bd-10690ebc31ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:49:04.122773503Z\",\n \"endTime\": \"2026-10-28T21:18:10.583197461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, packageserver, certified-operators-catalog, konnectivity-agent, cluster-policy-controller, cluster-node-tuning-operator, openshift-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator, olm-operator, community-operators-catalog, openshift-route-controller-manager, redhat-operators-catalog, redhat-marketplace-catalog, kube-scheduler, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-version-operator, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\n correlation ID: dfe646a0-0df6-424d-ac15-19436441fd56\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x2190c42a2260\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nd5rwvspn9b8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\",\n \"name\": \"71207653-6aed-4ccc-a2bd-10690ebc31ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:49:04.122773503Z\",\n \"endTime\": \"2026-10-28T21:18:10.583197461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, packageserver, certified-operators-catalog, konnectivity-agent, cluster-policy-controller, cluster-node-tuning-operator, openshift-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator, olm-operator, community-operators-catalog, openshift-route-controller-manager, redhat-operators-catalog, redhat-marketplace-catalog, kube-scheduler, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-version-operator, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71207653-6aed-4ccc-a2bd-10690ebc31ba\n correlation ID: dfe646a0-0df6-424d-ac15-19436441fd56\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2e390fa3e630965ae5daf0ecf63ae6fcb9584bc9589185875a776623474a507f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"5a2acf40c621f89bf24df745acf1ec5ca1bec8954c5e1db840e0c4501fae6ee4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-g2ccqb at version 4.22.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d077ee36720\u003e: \n failed to create HCP cluster np-skip-minor-g2ccqb: failed waiting for cluster=\"np-skip-minor-g2ccqb\" in resourcegroup=\"rg-np-skip-minor-g2ccqb-mn87dp5nj29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\",\n \"name\": \"1799e75d-c12f-41af-9c14-45990959eb21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:08.441802556Z\",\n \"endTime\": \"2026-10-28T21:26:10.61116359Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, cluster-network-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, cluster-policy-controller, konnectivity-agent, cluster-storage-operator, openshift-controller-manager, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, dns-operator, kube-storage-version-migrator, catalog-operator, community-operators-catalog, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\n correlation ID: 79286a08-358d-49d2-868d-a28ffe41cd8c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-g2ccqb at version 4.22.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d077ee36720\u003e: \n failed to create HCP cluster np-skip-minor-g2ccqb: failed waiting for cluster=\"np-skip-minor-g2ccqb\" in resourcegroup=\"rg-np-skip-minor-g2ccqb-mn87dp5nj29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\",\n \"name\": \"1799e75d-c12f-41af-9c14-45990959eb21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:08.441802556Z\",\n \"endTime\": \"2026-10-28T21:26:10.61116359Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, cluster-network-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, cluster-policy-controller, konnectivity-agent, cluster-storage-operator, openshift-controller-manager, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, dns-operator, kube-storage-version-migrator, catalog-operator, community-operators-catalog, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1799e75d-c12f-41af-9c14-45990959eb21\n correlation ID: 79286a08-358d-49d2-868d-a28ffe41cd8c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2e56dc60a816f81bc254428f7d116428db7e624eacb8136d52de0789d21d71ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"b6ec2d0660375e02ba6f95200ef4f85ae3f323412ef6a27f3400068636542d7f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-g7572t\" with version 4.21.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x5e791d44820\u003e: \n failed to create HCP cluster cluster-zstream-4-21-g7572t: failed waiting for cluster=\"cluster-zstream-4-21-g7572t\" in resourcegroup=\"rg-zstream-upgrade-4-21-9k9lb5xrt68d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\",\n \"name\": \"cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:53.702291365Z\",\n \"endTime\": \"2026-10-28T20:39:55.046264536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, redhat-marketplace-catalog, machine-approver, kube-scheduler, cluster-autoscaler, openshift-apiserver, azure-cloud-controller-manager, community-operators-catalog, openshift-controller-manager, ingress-operator, control-plane-pki-operator, cluster-network-operator, packageserver, olm-collect-profiles, certified-operators-catalog, cluster-policy-controller, cluster-storage-operator, cluster-image-registry-operator, olm-operator, kube-controller-manager, cluster-version-operator, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\n correlation ID: 75d43cba-42bf-4725-aaf6-4b8f69707172\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-g7572t\" with version 4.21.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x5e791d44820\u003e: \n failed to create HCP cluster cluster-zstream-4-21-g7572t: failed waiting for cluster=\"cluster-zstream-4-21-g7572t\" in resourcegroup=\"rg-zstream-upgrade-4-21-9k9lb5xrt68d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\",\n \"name\": \"cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:53.702291365Z\",\n \"endTime\": \"2026-10-28T20:39:55.046264536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, redhat-marketplace-catalog, machine-approver, kube-scheduler, cluster-autoscaler, openshift-apiserver, azure-cloud-controller-manager, community-operators-catalog, openshift-controller-manager, ingress-operator, control-plane-pki-operator, cluster-network-operator, packageserver, olm-collect-profiles, certified-operators-catalog, cluster-policy-controller, cluster-storage-operator, cluster-image-registry-operator, olm-operator, kube-controller-manager, cluster-version-operator, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd19c6e4-5a58-48c7-894c-aa0a8d05d58c\n correlation ID: 75d43cba-42bf-4725-aaf6-4b8f69707172\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"335b64505f12791300a634b60ef39cb561fa56dc9ac862352f2041efd51b8b44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"26c36401fd91e0588393f9c87cccfc2015e7a28fc80e00ea485b3b3fa225f6ea","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x1683f5097100\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mhvkjzh45hwc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n \"name\": \"e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:18.023653922Z\",\n \"endTime\": \"2026-10-28T19:57:23.841118908Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n correlation ID: 7d36723c-bee1-4adc-86af-798e4647be9f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x1683f5097100\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mhvkjzh45hwc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n \"name\": \"e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:18.023653922Z\",\n \"endTime\": \"2026-10-28T19:57:23.841118908Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n correlation ID: 7d36723c-bee1-4adc-86af-798e4647be9f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3ee7814add48569ed54e326d9db4ad1dce0d7dd1125868109cc18c13eef93bd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"762fee71e1d321f9cbabc110e98c51820d49a46e2eb414f6e17c5eff0e5b261d","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x14a89c71a940\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tpg6t2mn29xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\",\n \"name\": \"35be8b72-7d8c-49bd-8ea2-93aca817ec52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:19.423180703Z\",\n \"endTime\": \"2026-10-28T19:53:23.854104536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\n correlation ID: 7e7906b1-9aa5-44db-80a6-483bb9566fac\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x14a89c71a940\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tpg6t2mn29xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\",\n \"name\": \"35be8b72-7d8c-49bd-8ea2-93aca817ec52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:19.423180703Z\",\n \"endTime\": \"2026-10-28T19:53:23.854104536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/35be8b72-7d8c-49bd-8ea2-93aca817ec52\n correlation ID: 7e7906b1-9aa5-44db-80a6-483bb9566fac\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"446c79ba1954ef153058f4f9ecfa5bc654f66b40da0f91481c0845a6519c7c5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"1138e4fe25bc2fa499e07ce2095ef4b8e91229a94bc453bef3ca8e9f2be7f2bc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-jshv8zgq8wk9/cluster-candidate-4-23-c6gbbp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xba6b52aa80\u003e: \n failed to create HCP cluster cluster-candidate-4-23-c6gbbp: failed waiting for cluster=\"cluster-candidate-4-23-c6gbbp\" in resourcegroup=\"rg-candidate-4-23-jshv8zgq8wk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n \"name\": \"7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:20.02531103Z\",\n \"endTime\": \"2026-10-28T20:39:25.022579343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n correlation ID: 15324628-570b-42b9-80a7-dbd435939211\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-jshv8zgq8wk9/cluster-candidate-4-23-c6gbbp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xba6b52aa80\u003e: \n failed to create HCP cluster cluster-candidate-4-23-c6gbbp: failed waiting for cluster=\"cluster-candidate-4-23-c6gbbp\" in resourcegroup=\"rg-candidate-4-23-jshv8zgq8wk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n \"name\": \"7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:20.02531103Z\",\n \"endTime\": \"2026-10-28T20:39:25.022579343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n correlation ID: 15324628-570b-42b9-80a7-dbd435939211\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"462b52f6a620b30e62b594104aff4e87919e92a0c8fd18c17f807a15dc595fc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"b15ce3ecaa99dc5910d078a730555cb5295e82eb2dc3f11ddddbfa3af5dd845c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-21-mjsj484l4cg7/cluster-candidate-4-21-zqjdxv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e94b832e820\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zqjdxv: failed waiting for cluster=\"cluster-candidate-4-21-zqjdxv\" in resourcegroup=\"rg-candidate-4-21-mjsj484l4cg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n \"name\": \"e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:06:06.856819198Z\",\n \"endTime\": \"2026-10-28T20:35:15.000751357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-storage-operator, community-operators-catalog, ingress-operator, olm-operator, cluster-network-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, olm-collect-profiles, openshift-route-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n correlation ID: 0c3580d1-f96a-47b1-aaec-1d49dda7d990\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-21-mjsj484l4cg7/cluster-candidate-4-21-zqjdxv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e94b832e820\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zqjdxv: failed waiting for cluster=\"cluster-candidate-4-21-zqjdxv\" in resourcegroup=\"rg-candidate-4-21-mjsj484l4cg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n \"name\": \"e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:06:06.856819198Z\",\n \"endTime\": \"2026-10-28T20:35:15.000751357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-storage-operator, community-operators-catalog, ingress-operator, olm-operator, cluster-network-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, olm-collect-profiles, openshift-route-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n correlation ID: 0c3580d1-f96a-47b1-aaec-1d49dda7d990\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"48cfad62ec2eed4481da3e728fa297f5267b2cd4248bf6c05bdae732b7a4f87e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"71c974d1a4355b686678eaeec937bb1114fad8167d13110b0bd5deddaafdb12a","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c03b21d6c20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7fv9wkcmzdx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n \"name\": \"7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:37.658468995Z\",\n \"endTime\": \"2026-10-28T20:43:44.89685741Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n correlation ID: 1b096272-5f83-459d-a9b3-80242ffba49d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c03b21d6c20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7fv9wkcmzdx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n \"name\": \"7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:37.658468995Z\",\n \"endTime\": \"2026-10-28T20:43:44.89685741Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n correlation ID: 1b096272-5f83-459d-a9b3-80242ffba49d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"50dcfdd835d2934c674c97db7dc1e6c7295d0d3179534b8a906e0ed5e85caa66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"f2b41f64321f4d0c69f3e2ba802b7f90dfb2b447885bb1b355be09c939d2149d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-xkwqdc\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dceb2f8ab20\u003e: \n failed to create HCP cluster cluster-zstream-4-23-xkwqdc: failed waiting for cluster=\"cluster-zstream-4-23-xkwqdc\" in resourcegroup=\"rg-zstream-upgrade-4-23-blql6slsnrzk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\",\n \"name\": \"0020dfe1-eb87-4673-bb31-be80a802433f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:05.420114587Z\",\n \"endTime\": \"2026-10-28T20:40:15.034303339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, dns-operator, konnectivity-agent, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, hosted-cluster-config-operator, openshift-controller-manager, cluster-storage-operator, ingress-operator, packageserver, community-operators-catalog, kube-controller-manager, cluster-image-registry-operator, csi-snapshot-controller-operator, olm-operator, certified-operators-catalog, cluster-policy-controller, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\n correlation ID: cc2d8ce8-0861-418e-90fe-22edf335d211\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-xkwqdc\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dceb2f8ab20\u003e: \n failed to create HCP cluster cluster-zstream-4-23-xkwqdc: failed waiting for cluster=\"cluster-zstream-4-23-xkwqdc\" in resourcegroup=\"rg-zstream-upgrade-4-23-blql6slsnrzk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\",\n \"name\": \"0020dfe1-eb87-4673-bb31-be80a802433f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:05.420114587Z\",\n \"endTime\": \"2026-10-28T20:40:15.034303339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, dns-operator, konnectivity-agent, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, hosted-cluster-config-operator, openshift-controller-manager, cluster-storage-operator, ingress-operator, packageserver, community-operators-catalog, kube-controller-manager, cluster-image-registry-operator, csi-snapshot-controller-operator, olm-operator, certified-operators-catalog, cluster-policy-controller, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0020dfe1-eb87-4673-bb31-be80a802433f\n correlation ID: cc2d8ce8-0861-418e-90fe-22edf335d211\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"57bb41d24717f038f3d39b0b878d0782b6a8aa5f76f401a1ed1b1538701be6d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"b83842f62725a03eea143a87578ec93c1d0d4e416232fa1b48dba6d93d658a61","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-crm2px\" with version 4.20.43 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x5d990898640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-crm2px: failed waiting for cluster=\"cluster-zstream-4-20-crm2px\" in resourcegroup=\"rg-zstream-upgrade-4-20-ktl4cbqz8hgf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\",\n \"name\": \"227090e2-0a2d-418d-b6f5-f3e92ddc87ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:00.297311043Z\",\n \"endTime\": \"2026-10-28T20:40:05.017131119Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-marketplace-catalog, certified-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-policy-controller, catalog-operator, packageserver, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, capi-provider, control-plane-pki-operator, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, kube-controller-manager, openshift-apiserver, openshift-route-controller-manager, community-operators-catalog, cluster-autoscaler, cluster-network-operator, dns-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\n correlation ID: a0e92328-63d3-4953-90ed-c8b08db12589\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-crm2px\" with version 4.20.43 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x5d990898640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-crm2px: failed waiting for cluster=\"cluster-zstream-4-20-crm2px\" in resourcegroup=\"rg-zstream-upgrade-4-20-ktl4cbqz8hgf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\",\n \"name\": \"227090e2-0a2d-418d-b6f5-f3e92ddc87ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:00.297311043Z\",\n \"endTime\": \"2026-10-28T20:40:05.017131119Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-marketplace-catalog, certified-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-policy-controller, catalog-operator, packageserver, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, capi-provider, control-plane-pki-operator, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, kube-controller-manager, openshift-apiserver, openshift-route-controller-manager, community-operators-catalog, cluster-autoscaler, cluster-network-operator, dns-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/227090e2-0a2d-418d-b6f5-f3e92ddc87ca\n correlation ID: a0e92328-63d3-4953-90ed-c8b08db12589\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5ab3d5167b88d7a64805b5afb9f1c153b441191cf504a8e968d3f6cf9756b4db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"4b557052967c85109b6b45ef2f0fcb35fc9fc53f6bfbe6903a149cbafc57df96","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a9a9c302780\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dq6rz78qbzwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n \"name\": \"36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:46.976794807Z\",\n \"endTime\": \"2026-10-28T20:43:55.053369978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, ingress-operator, community-operators-catalog, dns-operator, packageserver, openshift-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n correlation ID: 0302f0c3-0eb3-48dd-9510-05b3665bc29f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a9a9c302780\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dq6rz78qbzwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n \"name\": \"36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:46.976794807Z\",\n \"endTime\": \"2026-10-28T20:43:55.053369978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, ingress-operator, community-operators-catalog, dns-operator, packageserver, openshift-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n correlation ID: 0302f0c3-0eb3-48dd-9510-05b3665bc29f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5ba2c1de6d155ab7b77cd8aabbaac28ed60b5f3afa4e30f163398cdb0e224685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"61abbe86523cd0cbbe34f1a7d4e1334ee3a9f32d67628855d0ff113bf0b2c7b8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lrsdfl\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c350f225e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lrsdfl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\",\n \"name\": \"034eed87-a977-47cf-8ff5-2b8292aa4616\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:34.208698264Z\",\n \"endTime\": \"2026-10-28T20:41:35.072616907Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lrsdfl\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c350f225e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lrsdfl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\",\n \"name\": \"034eed87-a977-47cf-8ff5-2b8292aa4616\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:34.208698264Z\",\n \"endTime\": \"2026-10-28T20:41:35.072616907Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"67e9fc5bf9bf4d2f0c861826c87e4b210b6e9b3f314720e2bde20267a3742bab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"36b9c2b22b1c0df002e2620d991914568e0046634273873e7ab53eaaf96416c7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x118c82e68960\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5fcqfhg6xm85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n \"name\": \"241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:58:03.268943167Z\",\n \"endTime\": \"2026-10-28T21:27:10.630246232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, catalog-operator, dns-operator, olm-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-policy-controller, packageserver, machine-approver, openshift-controller-manager, cluster-storage-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n correlation ID: 3553f468-7cef-45de-9639-6e7d0ba9a86e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x118c82e68960\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5fcqfhg6xm85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n \"name\": \"241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:58:03.268943167Z\",\n \"endTime\": \"2026-10-28T21:27:10.630246232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, catalog-operator, dns-operator, olm-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-policy-controller, packageserver, machine-approver, openshift-controller-manager, cluster-storage-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n correlation ID: 3553f468-7cef-45de-9639-6e7d0ba9a86e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"69cc75d4287040a508633e6c4d6698ac0c242723f6a720652a7315235c6ed84c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"154d3925be5c012a5113ed89b1b462845af15f57fbfd8e755b22fb15a8b69ce5","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x22868c3a2660\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s6kbbzpqdbx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n \"name\": \"a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:09.588021132Z\",\n \"endTime\": \"2026-10-28T19:53:13.758028769Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n correlation ID: 8f02eb4e-1e36-4dd4-b460-7588fb948e71\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x22868c3a2660\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s6kbbzpqdbx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n \"name\": \"a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:09.588021132Z\",\n \"endTime\": \"2026-10-28T19:53:13.758028769Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n correlation ID: 8f02eb4e-1e36-4dd4-b460-7588fb948e71\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6aeae8337611f44903b96bf62b7faf354360503636cfac66a98557ba6f3f6a3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"46ada0aa48fab5345ad452546fa116726a981cc504e0cec6c061ca9182ae0b45","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xfa629f24ea0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zkznqc7xhzd4\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-zkznqc7xhzd4\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xfa629c70d20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xfa629f24ea0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zkznqc7xhzd4\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-zkznqc7xhzd4\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xfa629c70d20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded"}},{"row_id":"6eee3dadbf9c043f21e717bcfe585e6bddce4072c59185c46f1314cce2167290","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"af33a2dad0cf09fb287a7b4234fdf373fb47cb8419d2c52ae3c2a78d006a72e2","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1308b97a65a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-79mst7jcpg45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n \"name\": \"94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:58:09.127282806Z\",\n \"endTime\": \"2026-10-28T21:27:10.682711777Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n correlation ID: 49b88924-064f-4f52-9480-109cf607a390\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1308b97a65a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-79mst7jcpg45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n \"name\": \"94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:58:09.127282806Z\",\n \"endTime\": \"2026-10-28T21:27:10.682711777Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n correlation ID: 49b88924-064f-4f52-9480-109cf607a390\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9114d91c0b1a3cf2341a472ef6ddec9850e63302a0009ffa178fc2dbceb63d9f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"72d4435f29601151d20deee599a6dee93d6fbe585b8ede64ef4f1b8cd77cf3bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"e374f7b8196855ef661f3e1f5e456bd341e5c59f7cf586f6925e971d4f0eceaa","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x2840a3490800\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9tjwn77pblt6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n \"name\": \"f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:48:29.562853635Z\",\n \"endTime\": \"2026-10-28T21:17:30.601011159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, konnectivity-agent, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n correlation ID: 12f04724-ca79-49ae-ac94-baef82ad2b08\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x2840a3490800\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9tjwn77pblt6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n \"name\": \"f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:48:29.562853635Z\",\n \"endTime\": \"2026-10-28T21:17:30.601011159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, konnectivity-agent, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n correlation ID: 12f04724-ca79-49ae-ac94-baef82ad2b08\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"04fa2dff8b6b51d28d362de7ef46f5028d2496b5105a9c36a86aac17ab5e65e0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"78313bc5421fc6b456eb6f891e98b75017997db809221a1358093bf8957e7168","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"9ac7fd28791c8e7cfc0f7d973ab1a8ddc1ce8e355b2f754b95756b7b44066b8a","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x23a60cd6e680\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ggtvxjcb6gls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n \"name\": \"6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:25.094565507Z\",\n \"endTime\": \"2026-10-28T19:53:33.692195544Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-autoscaler, olm-collect-profiles, redhat-operators-catalog, kube-scheduler, ingress-operator, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, olm-operator, capi-provider, openshift-apiserver, csi-snapshot-controller-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, dns-operator, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, packageserver, openshift-route-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n correlation ID: fb16cf1e-d070-4dc2-b0ad-1f7ce0d9da44\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x23a60cd6e680\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ggtvxjcb6gls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n \"name\": \"6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:25.094565507Z\",\n \"endTime\": \"2026-10-28T19:53:33.692195544Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-autoscaler, olm-collect-profiles, redhat-operators-catalog, kube-scheduler, ingress-operator, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, olm-operator, capi-provider, openshift-apiserver, csi-snapshot-controller-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, dns-operator, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, packageserver, openshift-route-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n correlation ID: fb16cf1e-d070-4dc2-b0ad-1f7ce0d9da44\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7c4b718ea856befdd05a6dfa093f4d65eacedb4da9e78273822d6a4852e1447b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ce6bdc393948e6a40a02a08779ba758155c6f242cf794d5aedb733749470627f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d947782ea20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w5rx25h5j8dw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n \"name\": \"0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:01:08.393167298Z\",\n \"endTime\": \"2026-10-28T20:30:16.146752142Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n correlation ID: 1eb4abcf-932d-4da4-828c-22172d8177f5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d947782ea20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w5rx25h5j8dw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n \"name\": \"0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:01:08.393167298Z\",\n \"endTime\": \"2026-10-28T20:30:16.146752142Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n correlation ID: 1eb4abcf-932d-4da4-828c-22172d8177f5\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0377bb07338fe0f98769c18440cfba81bee2c9117babcbbc37fcf8db2825921a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7d6748cf23025b4f2dba58203ae8a13eda8eae8eddf45528ee22e7c128463d02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"7bf017f71b8954e81509c7a98e71ca2c7f3c8fb5fa6883390b6c8cb356446a5a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d00e53ba4e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mpx2klxjj7k9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n \"name\": \"3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:17.131948207Z\",\n \"endTime\": \"2026-10-28T19:53:23.815769706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n correlation ID: c1be684a-2dcb-4ac2-af67-f7bd12ce1f70\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d00e53ba4e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mpx2klxjj7k9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n \"name\": \"3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:17.131948207Z\",\n \"endTime\": \"2026-10-28T19:53:23.815769706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n correlation ID: c1be684a-2dcb-4ac2-af67-f7bd12ce1f70\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8024ad31c60da00250d5959513d3f7a5cc09d2456fd8cd00bfbf416cecdb762f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"86b4b456f2d26e796b4d607ce207bc842c8f58c63934fa53482a3a51053ae023","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ca8efaeb8e5ba4fae1aa48de2092e9d863408e6f54ac717a04257cf2b20d4e5d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cf3d70a0be0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vjtmsffknq7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\",\n \"name\": \"44acf888-443c-4577-88c8-87d955fdc85d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:30.864672464Z\",\n \"endTime\": \"2026-10-28T19:53:33.757605836Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 24m49s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\n correlation ID: e7629f4b-1e00-4dc0-b99d-16a5be46a759\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cf3d70a0be0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vjtmsffknq7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\",\n \"name\": \"44acf888-443c-4577-88c8-87d955fdc85d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:30.864672464Z\",\n \"endTime\": \"2026-10-28T19:53:33.757605836Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 24m49s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44acf888-443c-4577-88c8-87d955fdc85d\n correlation ID: e7629f4b-1e00-4dc0-b99d-16a5be46a759\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"88c46ba9005f1ad501ac432a0bbfa1bf62ead69dc7dfa65fcbe1261acd4d3bad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"86b26fa2d9dd9e5aa9c08e0aee435bdda933b68e0c00ce13c0df48be507bb165","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2df91d94ebc0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c9fcp5td85qq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n \"name\": \"57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:52:14.429975606Z\",\n \"endTime\": \"2026-10-28T20:21:16.149275515Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n correlation ID: 7836a1e8-ab2a-417c-9954-360298a60ae2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2df91d94ebc0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c9fcp5td85qq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n \"name\": \"57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:52:14.429975606Z\",\n \"endTime\": \"2026-10-28T20:21:16.149275515Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n correlation ID: 7836a1e8-ab2a-417c-9954-360298a60ae2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8b42cc184a604f46c42a5d6f8401f924f874ae6e86b511cad9596f17ce45cdb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"4a6b63530c28e450b02962bbb0c52b15148d8644e14e86ec9e30122dd2845e15","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-dn5ff6 with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x9790da22600\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-dn5ff6: failed waiting for cluster=\"np-version-upgrade-cluster-dn5ff6\" in resourcegroup=\"rg-np-version-upgrade-dn5ff6-gc44wf2l4jv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n \"name\": \"e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:55:25.009646036Z\",\n \"endTime\": \"2026-10-28T21:24:30.625209621Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, ingress-operator, catalog-operator, kube-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n correlation ID: 9f77dae1-d4df-43a9-aae8-bcfb3bc64031\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-dn5ff6 with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x9790da22600\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-dn5ff6: failed waiting for cluster=\"np-version-upgrade-cluster-dn5ff6\" in resourcegroup=\"rg-np-version-upgrade-dn5ff6-gc44wf2l4jv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n \"name\": \"e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:55:25.009646036Z\",\n \"endTime\": \"2026-10-28T21:24:30.625209621Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, ingress-operator, catalog-operator, kube-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n correlation ID: 9f77dae1-d4df-43a9-aae8-bcfb3bc64031\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"908fe52767a4acb618025b654492709a258a8fb12ab3ac26379e38ab0e1705b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"2f48da34e6b9ed406b9605ee5e696280d567d993111a0dd4f10c0d136d538705","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc983e72b20\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h8vnhnh28g7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\",\n \"name\": \"338180fd-fd37-4118-8ce5-da19d007d2d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.13733405Z\",\n \"endTime\": \"2026-10-28T19:53:23.880336461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-version-operator, kube-controller-manager, catalog-operator, openshift-controller-manager, dns-operator, olm-operator, cluster-network-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, konnectivity-agent, packageserver, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n correlation ID: 92c6fcaf-99bb-49ef-8360-6f963a51cd80\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc983e72b20\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h8vnhnh28g7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\",\n \"name\": \"338180fd-fd37-4118-8ce5-da19d007d2d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.13733405Z\",\n \"endTime\": \"2026-10-28T19:53:23.880336461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-version-operator, kube-controller-manager, catalog-operator, openshift-controller-manager, dns-operator, olm-operator, cluster-network-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, konnectivity-agent, packageserver, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n correlation ID: 92c6fcaf-99bb-49ef-8360-6f963a51cd80\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9389ee237bf4022fe637a7801330b7ce4fe6dd2d99bc45ed683d3c8cb778f0f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"1c7eaaca2159b4db61db1cbd5d59ac643c49c639ef01458521ea8976229989c7","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x14a5e2008300\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k46j9j85xq5z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\",\n \"name\": \"a961a70a-5b5a-4175-bb58-82fb36beb466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:12.743400806Z\",\n \"endTime\": \"2026-10-28T19:55:13.7057976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n correlation ID: 5835c446-bdab-409a-8175-cbf5109c1943\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x14a5e2008300\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k46j9j85xq5z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\",\n \"name\": \"a961a70a-5b5a-4175-bb58-82fb36beb466\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:12.743400806Z\",\n \"endTime\": \"2026-10-28T19:55:13.7057976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n correlation ID: 5835c446-bdab-409a-8175-cbf5109c1943\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9f0e47bcbbc035fed5c0e4fd1c2e2d88f6eaf90bea1b91e2e9d78be47dcbf080","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"90762cacf1fc23e458f3602d5ca3c418a070b1c0b1de0b8c373357dcb930302c","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x110402dd07a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-l4d74x5hc8gz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\",\n \"name\": \"6987ce55-b0fd-4dcd-9644-01b84a500116\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:27:48.339569357Z\",\n \"endTime\": \"2026-10-28T19:56:53.862034082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\n correlation ID: 324dd373-c202-4e02-bc01-3d2aa84e7912\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x110402dd07a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-l4d74x5hc8gz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\",\n \"name\": \"6987ce55-b0fd-4dcd-9644-01b84a500116\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:27:48.339569357Z\",\n \"endTime\": \"2026-10-28T19:56:53.862034082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6987ce55-b0fd-4dcd-9644-01b84a500116\n correlation ID: 324dd373-c202-4e02-bc01-3d2aa84e7912\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a136a3d2d2b51d1d30c86e2d22ab5094eb6dbc08c386b07fc66118c1fc208edb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"a8a2dc5232fbd8f841fd16941f0452da6dddb6c06224f8a02d8ea593a401d9d8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1da2d78427a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r4m67m749clh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n \"name\": \"ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:05.081696809Z\",\n \"endTime\": \"2026-10-28T19:55:13.864538233Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n correlation ID: 259a0ca4-d067-4227-b0b5-f5f22a7f7fe0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1da2d78427a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r4m67m749clh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n \"name\": \"ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:05.081696809Z\",\n \"endTime\": \"2026-10-28T19:55:13.864538233Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n correlation ID: 259a0ca4-d067-4227-b0b5-f5f22a7f7fe0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a29316d25c372b2a11f0dd79cb816d1e8ded6c68ef49791f7593e87176279e53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"60d37d8dfdde6c0d2860f3361a3eb5f446365a64828f748da647a89a187b926b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-0-k2brb9cchnfb/cluster-candidate-5-0-c64hdj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x104ef3fa7e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-c64hdj: failed waiting for cluster=\"cluster-candidate-5-0-c64hdj\" in resourcegroup=\"rg-candidate-5-0-k2brb9cchnfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n \"name\": \"4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:48.049950576Z\",\n \"endTime\": \"2026-10-28T20:39:55.034999436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n correlation ID: b9bdaba2-ec65-4fd2-abc1-4305bb14fac0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-0-k2brb9cchnfb/cluster-candidate-5-0-c64hdj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x104ef3fa7e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-c64hdj: failed waiting for cluster=\"cluster-candidate-5-0-c64hdj\" in resourcegroup=\"rg-candidate-5-0-k2brb9cchnfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n \"name\": \"4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:48.049950576Z\",\n \"endTime\": \"2026-10-28T20:39:55.034999436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n correlation ID: b9bdaba2-ec65-4fd2-abc1-4305bb14fac0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a38036a5d864763dbfcc21e9b3d9bc7a6f587cb4644119bdd5147f8ec742659d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"4d63e7a6f84ed9ae74bd797e00b40ae72a6ac8953997ae0f91886ae1595029e7","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3b3d00e688d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n \"name\": \"e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:59:31.361013478Z\",\n \"endTime\": \"2026-10-28T21:28:40.621642045Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, packageserver, dns-operator, ingress-operator, olm-operator, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, control-plane-pki-operator, openshift-route-controller-manager, konnectivity-agent, redhat-marketplace-catalog, community-operators-catalog, cluster-storage-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n correlation ID: caf44bbd-ef58-4e77-8074-5770ca1391e6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3b3d00e688d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n \"name\": \"e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:59:31.361013478Z\",\n \"endTime\": \"2026-10-28T21:28:40.621642045Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, packageserver, dns-operator, ingress-operator, olm-operator, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, control-plane-pki-operator, openshift-route-controller-manager, konnectivity-agent, redhat-marketplace-catalog, community-operators-catalog, cluster-storage-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n correlation ID: caf44bbd-ef58-4e77-8074-5770ca1391e6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a78c69584071b4a1879b8417e652aa35d48d1641837b12d461b32253c5b74274","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"85a55c54ead7202802afe2a93f941819e12a2e1d54029fa4318f6226ee9536dd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-sgk9ch\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x12b89f7eb40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-sgk9ch: failed waiting for cluster=\"cp-ystream-upgrade-4-22-sgk9ch\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-27jpzmw77lts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\",\n \"name\": \"14601d65-c9c2-4ca1-aaf0-80f2a475267f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:33.485207001Z\",\n \"endTime\": \"2026-10-28T20:41:35.087630634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, packageserver, redhat-operators-catalog, kube-scheduler, olm-operator, kube-controller-manager, cluster-storage-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog, ingress-operator, csi-snapshot-controller-operator, cluster-version-operator, dns-operator, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\n correlation ID: 0f2608cc-c139-4188-9e46-cb83d44b4b49\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-sgk9ch\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x12b89f7eb40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-sgk9ch: failed waiting for cluster=\"cp-ystream-upgrade-4-22-sgk9ch\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-27jpzmw77lts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\",\n \"name\": \"14601d65-c9c2-4ca1-aaf0-80f2a475267f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:33.485207001Z\",\n \"endTime\": \"2026-10-28T20:41:35.087630634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, packageserver, redhat-operators-catalog, kube-scheduler, olm-operator, kube-controller-manager, cluster-storage-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog, ingress-operator, csi-snapshot-controller-operator, cluster-version-operator, dns-operator, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/14601d65-c9c2-4ca1-aaf0-80f2a475267f\n correlation ID: 0f2608cc-c139-4188-9e46-cb83d44b4b49\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a902e1ec1b530becc479e3e39459f721db034aefb845f2b88dc02a05e3910906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"b6dabb2ae36ec121af2506af67029f1dd28ff29ddcc71f7d20abbd9a72a3223a","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-95f5bw with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c6ab1a4a4c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-95f5bw: failed waiting for cluster=\"np-version-upgrade-cluster-95f5bw\" in resourcegroup=\"rg-np-version-upgrade-95f5bw-mppp9bg5hxs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\",\n \"name\": \"13f26b26-2324-48a4-82df-d8b324b13996\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:52:21.783931077Z\",\n \"endTime\": \"2026-10-28T21:21:30.723052246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 26m5s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n correlation ID: 9f0bfbd0-f67d-4e88-9467-75878a8fe604\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-95f5bw with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c6ab1a4a4c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-95f5bw: failed waiting for cluster=\"np-version-upgrade-cluster-95f5bw\" in resourcegroup=\"rg-np-version-upgrade-95f5bw-mppp9bg5hxs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\",\n \"name\": \"13f26b26-2324-48a4-82df-d8b324b13996\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:52:21.783931077Z\",\n \"endTime\": \"2026-10-28T21:21:30.723052246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 26m5s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n correlation ID: 9f0bfbd0-f67d-4e88-9467-75878a8fe604\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a9580fa14f55fab9047afd2c65a6f5d5479545f919872610f1c68730acdf2dfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"a19a06c858cf5713c455799ef09e3b0264f855a25c80378a222e92d8f7250eb2","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34142edd6a40\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-7s64tm5p2pnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\",\n \"name\": \"f063aff6-fa6b-4397-9b14-f08322d800e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:14.723008935Z\",\n \"endTime\": \"2026-10-28T19:53:23.809094912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, capi-provider, cluster-version-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, olm-collect-profiles, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, packageserver, cluster-autoscaler, azure-cloud-controller-manager, olm-operator, redhat-marketplace-catalog, kube-controller-manager, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, cluster-policy-controller, community-operators-catalog, redhat-operators-catalog, control-plane-pki-operator, konnectivity-agent, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n correlation ID: 9da90882-319b-4147-94e3-9897b59ff90d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34142edd6a40\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-7s64tm5p2pnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\",\n \"name\": \"f063aff6-fa6b-4397-9b14-f08322d800e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:14.723008935Z\",\n \"endTime\": \"2026-10-28T19:53:23.809094912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, capi-provider, cluster-version-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, olm-collect-profiles, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, packageserver, cluster-autoscaler, azure-cloud-controller-manager, olm-operator, redhat-marketplace-catalog, kube-controller-manager, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, cluster-policy-controller, community-operators-catalog, redhat-operators-catalog, control-plane-pki-operator, konnectivity-agent, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n correlation ID: 9da90882-319b-4147-94e3-9897b59ff90d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b1357a1dddbccc37daa7e03e273add49e01930db61df80db3292f7a6ab17ec27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"1aa62d268d8e60f0844f153da0305ef176957779ea820d90f8034d253c93b02a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xe23c97da540\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bsr8ks7wrjr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n \"name\": \"3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:28:51.316103778Z\",\n \"endTime\": \"2026-10-28T20:57:55.004654509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m35s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n correlation ID: 5b00d586-2a8f-4b57-a9e1-84466b469fca\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xe23c97da540\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bsr8ks7wrjr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n \"name\": \"3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:28:51.316103778Z\",\n \"endTime\": \"2026-10-28T20:57:55.004654509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m35s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n correlation ID: 5b00d586-2a8f-4b57-a9e1-84466b469fca\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b1fa6ee7c514827c55564e7b297042f6383cf399f8402b9bbc470ceb83de8c91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"a580fa2d1a60bc0f4d0f383e974139db9c4e6ed9723aac1cfd38205ae56a2708","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x38e5d805a900\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w6g9pxqkxlb5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\",\n \"name\": \"8f038d0c-188e-4fd1-b9b6-4a92361ba19c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.372217471Z\",\n \"endTime\": \"2026-10-28T19:53:23.8370741Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\n correlation ID: 634adffc-fd16-4c0c-b46c-854f988d728f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x38e5d805a900\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w6g9pxqkxlb5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\",\n \"name\": \"8f038d0c-188e-4fd1-b9b6-4a92361ba19c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.372217471Z\",\n \"endTime\": \"2026-10-28T19:53:23.8370741Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8f038d0c-188e-4fd1-b9b6-4a92361ba19c\n correlation ID: 634adffc-fd16-4c0c-b46c-854f988d728f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b806459dd476dafaef09f2eb6c5fe863be996ec17fd10e72cd17f2b4d648bcff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"50bccbab0fbf4c60d6c637d2f3d0712ba62570bb31026881eea6864443501319","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d67385a4720\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-292fs2ltfn6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n \"name\": \"de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:18.787179838Z\",\n \"endTime\": \"2026-10-28T19:53:23.770490111Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, kube-controller-manager, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, capi-provider, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, cluster-storage-operator, olm-collect-profiles, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, kube-scheduler, openshift-apiserver, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, dns-operator, olm-operator, openshift-controller-manager, packageserver, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n correlation ID: 5f8d39c8-29ad-4bdc-8281-5d6497fadbe8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d67385a4720\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-292fs2ltfn6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n \"name\": \"de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:18.787179838Z\",\n \"endTime\": \"2026-10-28T19:53:23.770490111Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, kube-controller-manager, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, capi-provider, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, cluster-storage-operator, olm-collect-profiles, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, kube-scheduler, openshift-apiserver, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, dns-operator, olm-operator, openshift-controller-manager, packageserver, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n correlation ID: 5f8d39c8-29ad-4bdc-8281-5d6497fadbe8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b84c247da81386e61468fe137eb0c591979813a4fafa3f59209b8b8f3c94ee2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"5f6d04b8f2b2c5c630e47972114cb8f68ab1ab545e7516b6f7ff6f5109fd3680","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x364bf26c2280\u003e: \n failed to create HCP cluster np-dg-minor-82f95j: failed waiting for cluster=\"np-dg-minor-82f95j\" in resourcegroup=\"rg-np-dg-minor-82f95j-47wd4n6jxmd6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\",\n \"name\": \"4c41c18d-b7df-4635-a0c8-e62246a9f100\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:20.485792808Z\",\n \"endTime\": \"2026-10-28T21:26:20.677519522Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, certified-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, olm-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, packageserver, cluster-version-operator, csi-snapshot-controller-operator, kube-storage-version-migrator, kube-scheduler, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\n correlation ID: 68c0efa5-5e54-499c-9838-0520a414503e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x364bf26c2280\u003e: \n failed to create HCP cluster np-dg-minor-82f95j: failed waiting for cluster=\"np-dg-minor-82f95j\" in resourcegroup=\"rg-np-dg-minor-82f95j-47wd4n6jxmd6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\",\n \"name\": \"4c41c18d-b7df-4635-a0c8-e62246a9f100\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:20.485792808Z\",\n \"endTime\": \"2026-10-28T21:26:20.677519522Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, certified-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, olm-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, packageserver, cluster-version-operator, csi-snapshot-controller-operator, kube-storage-version-migrator, kube-scheduler, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c41c18d-b7df-4635-a0c8-e62246a9f100\n correlation ID: 68c0efa5-5e54-499c-9838-0520a414503e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b8fbbc7ca61de377467ef7d89503475890c94031177b62171cf604e0e4252e2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"5b7dc99f064f8216ed626ab763d7c2a790998f03c03f0d31c47ee72a9d07d7aa","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x10164f31ca00\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p669mb2thnqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n \"name\": \"f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:27:58.648367476Z\",\n \"endTime\": \"2026-10-28T19:57:03.816857754Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x10164f31ca00\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p669mb2thnqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n \"name\": \"f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:27:58.648367476Z\",\n \"endTime\": \"2026-10-28T19:57:03.816857754Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"be9920f381e8a84b4aa554168582d14945eca7f6e54a27b8ea8e054b89f04fd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"4f9202c02773536be9847760ee3cf57fdb731b7b1a80187ad25cf10ef4a097f6","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x16967f5def20\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7wppdvdd55v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\",\n \"name\": \"54478daa-47e5-4530-91c2-342cc29a1749\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.628677591Z\",\n \"endTime\": \"2026-10-28T19:53:23.838328082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.20 is Partial (want Completed), started 26m9s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n correlation ID: 7c6978d3-1a22-47a9-861d-b27f5d6a3f88\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x16967f5def20\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7wppdvdd55v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\",\n \"name\": \"54478daa-47e5-4530-91c2-342cc29a1749\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.628677591Z\",\n \"endTime\": \"2026-10-28T19:53:23.838328082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.20 is Partial (want Completed), started 26m9s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n correlation ID: 7c6978d3-1a22-47a9-861d-b27f5d6a3f88\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3ff2f99fe58989a0c5bb5b159d7555e4151b1a641a1e248b4a62a0022252a52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"1b8fcec76c5e298c087b6dd13ca411d382423c18b384cd3f5c84450ce6644b1c","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x28ad82ebac20\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-8dptc8vk8gtv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n \"name\": \"d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:01:11.67931549Z\",\n \"endTime\": \"2026-10-28T20:30:16.178438369Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n correlation ID: 66a835d6-6997-4516-8fbc-7fd19b5ac311\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x28ad82ebac20\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-8dptc8vk8gtv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n \"name\": \"d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:01:11.67931549Z\",\n \"endTime\": \"2026-10-28T20:30:16.178438369Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n correlation ID: 66a835d6-6997-4516-8fbc-7fd19b5ac311\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2a4dc40bdb155c7b5e7f0e2213ef4c3de0123c4f0e14b53fa40355751f06e680","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9e256136eeff47c43ed671479d7b1daff6135e75f605c89a7a68c33cc285c04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ef55f5ef8cc6c413d0c5d55981f0b6ab1b6f752dc03d5d60ed0a2e58fc37935b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-t46hvnqkdqvf/cluster-candidate-4-22-2nwg57 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xa071317e240\u003e: \n failed to create HCP cluster cluster-candidate-4-22-2nwg57: failed waiting for cluster=\"cluster-candidate-4-22-2nwg57\" in resourcegroup=\"rg-candidate-4-22-t46hvnqkdqvf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\",\n \"name\": \"fa78952a-2a0e-4c91-9b40-6d20c521b765\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:11.743652526Z\",\n \"endTime\": \"2026-10-28T20:39:15.054194298Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\n correlation ID: 3428fa79-40a9-4a5a-8766-7a9690285732\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-t46hvnqkdqvf/cluster-candidate-4-22-2nwg57 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xa071317e240\u003e: \n failed to create HCP cluster cluster-candidate-4-22-2nwg57: failed waiting for cluster=\"cluster-candidate-4-22-2nwg57\" in resourcegroup=\"rg-candidate-4-22-t46hvnqkdqvf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\",\n \"name\": \"fa78952a-2a0e-4c91-9b40-6d20c521b765\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:10:11.743652526Z\",\n \"endTime\": \"2026-10-28T20:39:15.054194298Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa78952a-2a0e-4c91-9b40-6d20c521b765\n correlation ID: 3428fa79-40a9-4a5a-8766-7a9690285732\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ccb64c4fb7202dbe9588917a34368b97ed64398651e5f49b686497897b430e96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ad80f1e1dfb5cb1f986c967a0119074ab300822bb8d46340e61875c7b780b23e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x38cab388f240\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-z6f4tl45wbl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\",\n \"name\": \"62ac4023-4fb6-42a6-87c4-4a745d49d80b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:41.52216937Z\",\n \"endTime\": \"2026-10-28T19:57:43.694575383Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\n correlation ID: 0bc7323c-15c6-4cfa-85ba-27c1c70f38cc\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x38cab388f240\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-z6f4tl45wbl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\",\n \"name\": \"62ac4023-4fb6-42a6-87c4-4a745d49d80b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:28:41.52216937Z\",\n \"endTime\": \"2026-10-28T19:57:43.694575383Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62ac4023-4fb6-42a6-87c4-4a745d49d80b\n correlation ID: 0bc7323c-15c6-4cfa-85ba-27c1c70f38cc\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d354712608d99fd7388af169e5d5f670be2bcaa7439c4e178d0a2e649cd2b9e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"68b0cfa89285503fbd77f7180bd24bbc791f53e770eaf8d7210f219c10f28a08","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-clmhlg\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d29037be7e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-clmhlg: failed waiting for cluster=\"cp-ystream-upgrade-5-1-clmhlg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-m7flj58khr4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\",\n \"name\": \"75f18780-46df-4ca4-b758-e34d2262ebd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:38.563887566Z\",\n \"endTime\": \"2026-10-28T20:41:44.890635353Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n correlation ID: ae6aa7e5-e845-4182-9588-f52bb0a88718\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-clmhlg\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d29037be7e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-clmhlg: failed waiting for cluster=\"cp-ystream-upgrade-5-1-clmhlg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-m7flj58khr4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\",\n \"name\": \"75f18780-46df-4ca4-b758-e34d2262ebd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:12:38.563887566Z\",\n \"endTime\": \"2026-10-28T20:41:44.890635353Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n correlation ID: ae6aa7e5-e845-4182-9588-f52bb0a88718\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfee1963b853a23949e2e783c609db8a42592015ad0fdf8e31566559d18f7443","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"3240f3cd2cf3f8b8a71103086ee8514b0eaf1e7b232f5cde5bc18a32a344ae68","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-swblz4\" with version 4.22.19 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3981bcd4a5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-swblz4: failed waiting for cluster=\"cluster-zstream-4-22-swblz4\" in resourcegroup=\"rg-zstream-upgrade-4-22-nkrsx2q2bfk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\",\n \"name\": \"fa758444-9da4-4261-b65a-fb9af0416aae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:05.048147318Z\",\n \"endTime\": \"2026-10-28T20:40:15.036753709Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-policy-controller, kube-controller-manager, cluster-version-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, kube-scheduler, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, ingress-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n correlation ID: 8fa02e62-e5c9-4192-8dc2-5346dc0423f9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-swblz4\" with version 4.22.19 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3981bcd4a5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-swblz4: failed waiting for cluster=\"cluster-zstream-4-22-swblz4\" in resourcegroup=\"rg-zstream-upgrade-4-22-nkrsx2q2bfk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\",\n \"name\": \"fa758444-9da4-4261-b65a-fb9af0416aae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:05.048147318Z\",\n \"endTime\": \"2026-10-28T20:40:15.036753709Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-policy-controller, kube-controller-manager, cluster-version-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, kube-scheduler, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, ingress-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n correlation ID: 8fa02e62-e5c9-4192-8dc2-5346dc0423f9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e22049035b2cb29e052cd91648293ebff4930c77268ba720358d59b56bce4682","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"a48400b513d98d37b9edb6f349d4f9b36b7a3f5371ab99314e6d5ace05ca1b4f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cz8ptt with version 4.20.44\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c7b6c0a8440\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cz8ptt: failed waiting for cluster=\"np-version-upgrade-cluster-cz8ptt\" in resourcegroup=\"rg-np-version-upgrade-cz8ptt-hnxbd7dxv6dv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n \"name\": \"6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:56:27.61784978Z\",\n \"endTime\": \"2026-10-28T21:25:30.606619249Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 25m13s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n correlation ID: 4dbfaa07-bbd8-4943-933b-1b427a38701f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cz8ptt with version 4.20.44\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c7b6c0a8440\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cz8ptt: failed waiting for cluster=\"np-version-upgrade-cluster-cz8ptt\" in resourcegroup=\"rg-np-version-upgrade-cz8ptt-hnxbd7dxv6dv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n \"name\": \"6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:56:27.61784978Z\",\n \"endTime\": \"2026-10-28T21:25:30.606619249Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 25m13s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n correlation ID: 4dbfaa07-bbd8-4943-933b-1b427a38701f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e34b47d2e3cb27da8f8dec3dae66a06cd5de9c377cd9d2cb4598bc745cd5c2e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"b89fb0e7cd2e1b39b36c5079ebdce4206fba71d4d1664e685853e2da6ae8985e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x385465998780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m8h7nbml5knf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\",\n \"name\": \"7c42a163-6c2b-42e8-a29b-4e6883ba3f36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:03.613135527Z\",\n \"endTime\": \"2026-10-28T19:55:03.867198108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\n correlation ID: 85b18aae-16d5-46e6-a585-e2682c6d10ca\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x385465998780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m8h7nbml5knf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\",\n \"name\": \"7c42a163-6c2b-42e8-a29b-4e6883ba3f36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:26:03.613135527Z\",\n \"endTime\": \"2026-10-28T19:55:03.867198108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c42a163-6c2b-42e8-a29b-4e6883ba3f36\n correlation ID: 85b18aae-16d5-46e6-a585-e2682c6d10ca\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e956b6298649c2de4c4a7c6952cf18f2f471d40f503c5d853fa4cdbdbbe86d3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"9fef6cdd46bcccbc965ee603bb9552f0ff376c7d7b5cc6240f49957e233a60ed","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3290e061a680\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qdsgxxj79bfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n \"name\": \"5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:27.436526224Z\",\n \"endTime\": \"2026-10-28T21:26:30.622804028Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, cluster-policy-controller, openshift-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n correlation ID: 2fd33249-a131-4a13-a1a2-03c755186865\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3290e061a680\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qdsgxxj79bfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n \"name\": \"5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:27.436526224Z\",\n \"endTime\": \"2026-10-28T21:26:30.622804028Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, cluster-policy-controller, openshift-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n correlation ID: 2fd33249-a131-4a13-a1a2-03c755186865\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b95030e680b85c36e813048d300579274516d1f1a72079e03a11261579e993f4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-scheduler, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"eaddf3e1507f3ed3d1707d70385aac0e011653f68280834ce33e99026ab0b17e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"ab061e66e29e6d1c2a90de4abf7a2dc3ca11982db8c3aca1c2d0b21771747f26","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x26ceb862cd00\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wz7pqrvxjxjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\",\n \"name\": \"94aab46e-a3d9-462d-bb43-0c5fc752748f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.25232077Z\",\n \"endTime\": \"2026-10-28T19:53:23.685636564Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\n correlation ID: 745d7020-48a3-43c5-8d75-45376466cb1f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x26ceb862cd00\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wz7pqrvxjxjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\",\n \"name\": \"94aab46e-a3d9-462d-bb43-0c5fc752748f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.25232077Z\",\n \"endTime\": \"2026-10-28T19:53:23.685636564Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94aab46e-a3d9-462d-bb43-0c5fc752748f\n correlation ID: 745d7020-48a3-43c5-8d75-45376466cb1f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"efa99d2d2eda840c7de77c18b4a1ce7f036fd45654654510b2a203e4cfcdf5a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"d347788cb0265beb929b34ab787405bf42d848f46684380adad1dd8be49a95ea","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3735652cc6a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nxtm6wfxc8r7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\",\n \"name\": \"38392dc4-2811-4cda-8591-7ffe1ffac7c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.68302357Z\",\n \"endTime\": \"2026-10-28T19:53:23.756141507Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, konnectivity-agent, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, openshift-controller-manager, ingress-operator, kube-scheduler, cluster-version-operator, csi-snapshot-controller-operator, kube-storage-version-migrator, certified-operators-catalog, kube-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, openshift-route-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, packageserver, community-operators-catalog, catalog-operator, machine-approver, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\n correlation ID: b11a943b-2f1f-4773-a692-76ceee856dc1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3735652cc6a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nxtm6wfxc8r7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\",\n \"name\": \"38392dc4-2811-4cda-8591-7ffe1ffac7c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.68302357Z\",\n \"endTime\": \"2026-10-28T19:53:23.756141507Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, konnectivity-agent, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, openshift-controller-manager, ingress-operator, kube-scheduler, cluster-version-operator, csi-snapshot-controller-operator, kube-storage-version-migrator, certified-operators-catalog, kube-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, openshift-route-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, packageserver, community-operators-catalog, catalog-operator, machine-approver, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38392dc4-2811-4cda-8591-7ffe1ffac7c1\n correlation ID: b11a943b-2f1f-4773-a692-76ceee856dc1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f390b291dec0df1f08889ad7a628c4ea646c1473ddbd4d7a7bc9a6f6f3870fb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"fa5c99ec12e75a03478d289817e89bb3f5f8869f0cee804d3097d4317f930455","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x44f1a4fe8a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jf8m6zdj6vs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\",\n \"name\": \"3346d2af-f75c-40f6-8165-8b6d194d6773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.343807929Z\",\n \"endTime\": \"2026-10-28T19:53:23.794639518Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\n correlation ID: 1abc5bd8-a67b-46a9-affe-3b9e66e6ca79\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x44f1a4fe8a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jf8m6zdj6vs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\",\n \"name\": \"3346d2af-f75c-40f6-8165-8b6d194d6773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:20.343807929Z\",\n \"endTime\": \"2026-10-28T19:53:23.794639518Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3346d2af-f75c-40f6-8165-8b6d194d6773\n correlation ID: 1abc5bd8-a67b-46a9-affe-3b9e66e6ca79\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f3d88babccf579abdc5f990608368a11dbe01985a67ff258a8e6437a7df6d4d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"5a4f21e5c2f30937e23a51b2fe564d63df2d5755f70f2cc3115bee5786c1490b","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x38a796b1a740\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lvjvqhcdtkrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\",\n \"name\": \"499291aa-b279-429d-80ba-7786fc6cfa93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:38:17.873619905Z\",\n \"endTime\": \"2026-10-28T21:07:24.99192406Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n correlation ID: 7577cb59-123a-459f-a8f3-f4a03306f15d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x38a796b1a740\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lvjvqhcdtkrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\",\n \"name\": \"499291aa-b279-429d-80ba-7786fc6cfa93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:38:17.873619905Z\",\n \"endTime\": \"2026-10-28T21:07:24.99192406Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n correlation ID: 7577cb59-123a-459f-a8f3-f4a03306f15d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f5b3388c64e68ba88dd985115318b74c9b821146d61c6a0b8c5be1d7b5ac95a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"44d9b2ce46d7f92b1a97a2b7006b43cfe018754c0a9d5a038f4bf77df49a7765","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x263f7a1ecb40\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s854pt5lzxq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\",\n \"name\": \"bc13db95-dad4-4bc2-9929-827244e95d2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:34.589492538Z\",\n \"endTime\": \"2026-10-28T20:43:34.88292258Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n correlation ID: da1e3131-8ea4-44cd-87a3-857957002869\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x263f7a1ecb40\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s854pt5lzxq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\",\n \"name\": \"bc13db95-dad4-4bc2-9929-827244e95d2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:14:34.589492538Z\",\n \"endTime\": \"2026-10-28T20:43:34.88292258Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n correlation ID: da1e3131-8ea4-44cd-87a3-857957002869\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f6beff094bac6b2349c83ca45e91359db3e57af3cbf07381c97dfb8a374fe4d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"78ae657176cafaf184588b18ae514d272d0816a3ee3c0b87eea4e62207620774","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c9f11d305c0\u003e: \n failed to create HCP cluster np-downgrade-pgblzc: failed waiting for cluster=\"np-downgrade-pgblzc\" in resourcegroup=\"rg-np-downgrade-pgblzc-w296czt7c4h5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n \"name\": \"c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:10.002650956Z\",\n \"endTime\": \"2026-10-28T21:26:10.63929862Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, cluster-version-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, kube-scheduler, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, dns-operator, cluster-image-registry-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n correlation ID: 3652d2cb-5ad7-44b4-9342-18162d59e0da\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c9f11d305c0\u003e: \n failed to create HCP cluster np-downgrade-pgblzc: failed waiting for cluster=\"np-downgrade-pgblzc\" in resourcegroup=\"rg-np-downgrade-pgblzc-w296czt7c4h5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n \"name\": \"c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:57:10.002650956Z\",\n \"endTime\": \"2026-10-28T21:26:10.63929862Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, cluster-version-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, kube-scheduler, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, dns-operator, cluster-image-registry-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n correlation ID: 3652d2cb-5ad7-44b4-9342-18162d59e0da\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1edd5970e96b87d9128b141b52d9d5df9ec69d99da03a6f49cf9571a2753ec7e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f7754ee4b7d224257994128c84dc8ebfbc13a027a1b6e65cc9d2ca60b5180273","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"83a55ef34f01654f6dc3600d12e4c084f0f9f1b138bd67071de6c9fc2819ebea","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-s9k5d8\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x219ad233aca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-s9k5d8: failed waiting for cluster=\"cluster-zstream-5-0-s9k5d8\" in resourcegroup=\"rg-zstream-upgrade-5-0-n6xzg4dztfw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\",\n \"name\": \"e768a2f1-bf75-4afe-9acf-741914b73091\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:31.970768227Z\",\n \"endTime\": \"2026-10-28T20:40:35.054714998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, kube-controller-manager, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, machine-approver, kube-scheduler, openshift-apiserver, openshift-controller-manager, cluster-image-registry-operator, catalog-operator, capi-provider, kube-storage-version-migrator, olm-operator, packageserver, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n correlation ID: 12e6213e-1673-4bde-bd5f-964565fbe8d0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-s9k5d8\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x219ad233aca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-s9k5d8: failed waiting for cluster=\"cluster-zstream-5-0-s9k5d8\" in resourcegroup=\"rg-zstream-upgrade-5-0-n6xzg4dztfw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\",\n \"name\": \"e768a2f1-bf75-4afe-9acf-741914b73091\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T20:11:31.970768227Z\",\n \"endTime\": \"2026-10-28T20:40:35.054714998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, kube-controller-manager, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, machine-approver, kube-scheduler, openshift-apiserver, openshift-controller-manager, cluster-image-registry-operator, catalog-operator, capi-provider, kube-storage-version-migrator, olm-operator, packageserver, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n correlation ID: 12e6213e-1673-4bde-bd5f-964565fbe8d0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bf28464fdc326495b2d6ca9e56df879b27c4e19498ca8f2c4dfd708b81c66524","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f893e599046ee27a358d7258239aa6733be881616af6d4fcd29c357040d29600","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","failed_at":"e2e","signature_id":"d25cb3f84c24fd62b741353ecaf77155f1270ddfcbac771250ea07b39d1a3825","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eab57386d00\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pq4dgm8h8d5m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\",\n \"name\": \"1b5b8c7b-0882-4b28-b36d-8d86306ba636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.125042883Z\",\n \"endTime\": \"2026-10-28T19:53:23.713000323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\n correlation ID: 3c010b6b-955b-427d-a23b-ecd8944c4077\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eab57386d00\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pq4dgm8h8d5m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\",\n \"name\": \"1b5b8c7b-0882-4b28-b36d-8d86306ba636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T19:24:21.125042883Z\",\n \"endTime\": \"2026-10-28T19:53:23.713000323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b5b8c7b-0882-4b28-b36d-8d86306ba636\n correlation ID: 3c010b6b-955b-427d-a23b-ecd8944c4077\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":63,"failure_pattern_ids":["0377bb07338fe0f98769c18440cfba81bee2c9117babcbbc37fcf8db2825921a","04fa2dff8b6b51d28d362de7ef46f5028d2496b5105a9c36a86aac17ab5e65e0","120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","1edd5970e96b87d9128b141b52d9d5df9ec69d99da03a6f49cf9571a2753ec7e","2a4dc40bdb155c7b5e7f0e2213ef4c3de0123c4f0e14b53fa40355751f06e680","37985da5f97bff5443a20ce7eddabdbfd55a0839dc6b22fe1cac002e46ccec2c","4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","9114d91c0b1a3cf2341a472ef6ddec9850e63302a0009ffa178fc2dbceb63d9f","94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","b95030e680b85c36e813048d300579274516d1f1a72079e03a11261579e993f4","bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","bf28464fdc326495b2d6ca9e56df879b27c4e19498ca8f2c4dfd708b81c66524","c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","facbcb79129e6159c9b5455a0c79d2a7fec07d572d42676eaad216f2de41ba83"],"matched_occurrences":63,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115511222152466432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T18:29:14Z","started_at":"2026-10-28T18:29:14Z","completed_at":"2026-10-28T20:35:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T20:41:44.878075025Z","timing_metadata_checked_at":"2026-10-28T20:41:44.878075025Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T20:41:44.881815113Z","region_metadata_checked_at":"2026-10-28T20:41:44.881815113Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T13:09:17Z","started_at":"2026-10-28T13:09:17Z","completed_at":"2026-10-28T16:21:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T16:28:20.823744984Z","timing_metadata_checked_at":"2026-10-28T16:28:20.823744984Z","region":"eastus2euap","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T16:28:20.827364473Z","region_metadata_checked_at":"2026-10-28T16:28:20.827364473Z"},"failed_at":["e2e"],"failed_test_count":66,"occurrences":[{"row_id":"040afa513e540e34011e0ade6a43c9a025eea13b88bad48e5d2d9290e844d6fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"cbc0103ee5a417fd440bec6956285feaa5bda8eafcb0c41d737a626acc594ca2","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f34eadb6e20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dbk6jgfrnzkj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\",\n \"name\": \"eacff0a3-4bd4-4c15-aafe-008fc8887810\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:17.380797935Z\",\n \"endTime\": \"2026-10-28T15:50:17.704324747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\n correlation ID: 3ef3ff1c-e195-4831-adde-5137de419b11\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f34eadb6e20\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dbk6jgfrnzkj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\",\n \"name\": \"eacff0a3-4bd4-4c15-aafe-008fc8887810\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:17.380797935Z\",\n \"endTime\": \"2026-10-28T15:50:17.704324747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eacff0a3-4bd4-4c15-aafe-008fc8887810\n correlation ID: 3ef3ff1c-e195-4831-adde-5137de419b11\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"05bd393bd49f584c02c25ab82609f510a1b8fac8fe42acb14565b3de3bcdef7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"2f0c8223166367ebb908c5e6dac5e21ad27bbd34413df694da368440c482b45c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b896839fa60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fld8q7cxfnnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\",\n \"name\": \"fad0e7c8-51f4-402f-b2c0-d5c286ac7592\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:44:11.609067289Z\",\n \"endTime\": \"2026-10-28T14:13:20.968792662Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\n correlation ID: 19e863b6-3287-4026-a228-cda73ba74c13\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b896839fa60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fld8q7cxfnnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\",\n \"name\": \"fad0e7c8-51f4-402f-b2c0-d5c286ac7592\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:44:11.609067289Z\",\n \"endTime\": \"2026-10-28T14:13:20.968792662Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad0e7c8-51f4-402f-b2c0-d5c286ac7592\n correlation ID: 19e863b6-3287-4026-a228-cda73ba74c13\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"066e346c1fe63d208a8ccd7555ae325da7619bad9dab6af6d9898211272efcc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"29f04c65df4c7248fa410a89d3babf8f77358c45cd2c52b5a6b3fc4d89ee052b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-p65ngr\" with version 4.22.19 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f72bfdcc60\u003e: \n failed to create HCP cluster cluster-zstream-4-22-p65ngr: failed waiting for cluster=\"cluster-zstream-4-22-p65ngr\" in resourcegroup=\"rg-zstream-upgrade-4-22-fk4qxqhpjj29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n \"name\": \"8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:56.97734875Z\",\n \"endTime\": \"2026-10-28T15:08:58.578863234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, ingress-operator, csi-snapshot-controller-operator, community-operators-catalog, dns-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n correlation ID: 094c53a6-4603-4b58-b443-8cca1dbc9dbb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-p65ngr\" with version 4.22.19 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f72bfdcc60\u003e: \n failed to create HCP cluster cluster-zstream-4-22-p65ngr: failed waiting for cluster=\"cluster-zstream-4-22-p65ngr\" in resourcegroup=\"rg-zstream-upgrade-4-22-fk4qxqhpjj29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n \"name\": \"8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:56.97734875Z\",\n \"endTime\": \"2026-10-28T15:08:58.578863234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, ingress-operator, csi-snapshot-controller-operator, community-operators-catalog, dns-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n correlation ID: 094c53a6-4603-4b58-b443-8cca1dbc9dbb\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0789aa1ab8c97b974fef6a8027f63ffed59c9f56ddf2c45aa0d6e62a86d3761f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"30ec818b40b7b823a54f8213ce2ad6d08f8cd9991b7021927d7167695678fdc7","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x167c557a6660\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wjkxs2684hd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\",\n \"name\": \"9a50ba0b-b907-416a-b2d1-94636b24ec6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:47.959137424Z\",\n \"endTime\": \"2026-10-28T14:11:50.925878227Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\n correlation ID: 9f148a4c-d977-4e05-985d-74dfc49e11a0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x167c557a6660\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wjkxs2684hd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\",\n \"name\": \"9a50ba0b-b907-416a-b2d1-94636b24ec6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:47.959137424Z\",\n \"endTime\": \"2026-10-28T14:11:50.925878227Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a50ba0b-b907-416a-b2d1-94636b24ec6d\n correlation ID: 9f148a4c-d977-4e05-985d-74dfc49e11a0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0a9938f7105045927b6e11feacda2b8fc1f575b2e29d9780cb79d28ee04ddd10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"6a88c826b274fcdb6aada26d363a44e43a7b4641e212999dd5097ed4710a8547","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x19f9bb4550e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwx565kbs7q4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\",\n \"name\": \"859613d5-8f1e-4a8a-b5fd-b35307b04c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.85183749Z\",\n \"endTime\": \"2026-10-28T15:50:07.785672722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, ingress-operator, control-plane-pki-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver, cluster-image-registry-operator, olm-operator, redhat-marketplace-catalog, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, community-operators-catalog, kube-controller-manager, catalog-operator, kube-scheduler, cluster-version-operator, hosted-cluster-config-operator, dns-operator, azure-cloud-controller-manager, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\n correlation ID: 0727a39d-6d6c-4b6d-9eb9-78345449ac83\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x19f9bb4550e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwx565kbs7q4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\",\n \"name\": \"859613d5-8f1e-4a8a-b5fd-b35307b04c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.85183749Z\",\n \"endTime\": \"2026-10-28T15:50:07.785672722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, ingress-operator, control-plane-pki-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver, cluster-image-registry-operator, olm-operator, redhat-marketplace-catalog, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, community-operators-catalog, kube-controller-manager, catalog-operator, kube-scheduler, cluster-version-operator, hosted-cluster-config-operator, dns-operator, azure-cloud-controller-manager, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859613d5-8f1e-4a8a-b5fd-b35307b04c99\n correlation ID: 0727a39d-6d6c-4b6d-9eb9-78345449ac83\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"157b36f01fb85d72279ab424aa4179be938b0879688496091569a33c5817485c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"690d9a94c995b86ed981ba30241c5465df4455efe83abc764eb104e0bd36d163","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-jj2xlg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1496482ec2a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-jj2xlg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-jj2xlg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gp9nmrbz6s6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\",\n \"name\": \"39fc987d-482c-4621-a697-4e7a3438eafa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:45.234461665Z\",\n \"endTime\": \"2026-10-28T15:09:48.451075236Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, packageserver, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, catalog-operator, olm-operator, certified-operators-catalog, cluster-version-operator, redhat-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, openshift-controller-manager, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\n correlation ID: b0c553ac-6da6-4f3f-b3d3-f91a300d8bd6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-jj2xlg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1496482ec2a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-jj2xlg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-jj2xlg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gp9nmrbz6s6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\",\n \"name\": \"39fc987d-482c-4621-a697-4e7a3438eafa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:45.234461665Z\",\n \"endTime\": \"2026-10-28T15:09:48.451075236Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, packageserver, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, catalog-operator, olm-operator, certified-operators-catalog, cluster-version-operator, redhat-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, openshift-controller-manager, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39fc987d-482c-4621-a697-4e7a3438eafa\n correlation ID: b0c553ac-6da6-4f3f-b3d3-f91a300d8bd6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"183cadb20c1b63023549387299c3e1d646aeded93993a779e3f7f1ad867a5494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"87410719fd670b54f3aa86b66921cd3b52e5224e64794b89568b61f7af484cbc","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7ea8232700\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xz8d5m7w5nl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\",\n \"name\": \"88c4fef2-0e08-45c2-8eb6-f41b487cd618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:44:49.504870916Z\",\n \"endTime\": \"2026-10-28T14:13:50.792209417Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\n correlation ID: 945c0bd0-a046-43be-9723-689e4959653f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7ea8232700\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xz8d5m7w5nl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\",\n \"name\": \"88c4fef2-0e08-45c2-8eb6-f41b487cd618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:44:49.504870916Z\",\n \"endTime\": \"2026-10-28T14:13:50.792209417Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88c4fef2-0e08-45c2-8eb6-f41b487cd618\n correlation ID: 945c0bd0-a046-43be-9723-689e4959653f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1a435714e7afeb18d4d0a36ab9c3b09d77e8b1ed6193176953cc25d90931a830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"d5a5380e019074b97f540ce8686a93a598d62b7d02fc1170ba11db74040bc9fe","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c6dd746e940\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-22sbv67qz5xz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\",\n \"name\": \"8d92ce31-2582-4a90-ac50-186535026c4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:56.490101151Z\",\n \"endTime\": \"2026-10-28T15:50:57.635126569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-version-operator, ingress-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-policy-controller, dns-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-image-registry-operator, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\n correlation ID: b90608ee-eef4-4f50-8721-a501ceb1dc3b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c6dd746e940\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-22sbv67qz5xz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\",\n \"name\": \"8d92ce31-2582-4a90-ac50-186535026c4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:56.490101151Z\",\n \"endTime\": \"2026-10-28T15:50:57.635126569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-version-operator, ingress-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-policy-controller, dns-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-image-registry-operator, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d92ce31-2582-4a90-ac50-186535026c4b\n correlation ID: b90608ee-eef4-4f50-8721-a501ceb1dc3b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1a8c3d3e2c5cf01c290e0af607173776e083b7203b735a2254d1aedfac8b8739","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"a7a82dbb2248dec4c46a4ed0be66a247d60fbf4440afa724a4ffcf4486ebe06a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x144b1f0220a8\u003e: \n resource group \"ingress-cert-le-pqlv4rswg88h\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pqlv4rswg88h\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n \"name\": \"f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:05:17.896484451Z\",\n \"endTime\": \"2026-10-28T14:29:22.39155595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tnqroht6vs8i90m1kekfo1uicveo895 still exists (deletion dispatched at 2026-10-28T14:05:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n correlation ID: 3972a876-9ac5-4cc1-a07c-3501cf7d75bd\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x144b1f0220a8\u003e: \n resource group \"ingress-cert-le-pqlv4rswg88h\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pqlv4rswg88h\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n \"name\": \"f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:05:17.896484451Z\",\n \"endTime\": \"2026-10-28T14:29:22.39155595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tnqroht6vs8i90m1kekfo1uicveo895 still exists (deletion dispatched at 2026-10-28T14:05:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n correlation ID: 3972a876-9ac5-4cc1-a07c-3501cf7d75bd\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1be7803025145888aa9815413b907af55a5256a357d22b0aa4f3c5a783d90293","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"8476042b822d47e0f54de1129ee15563dd457884f1919c8e971a1c4b89973cad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-21-6v7gsgmssgvq/cluster-candidate-4-21-rh49gm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1da958962480\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rh49gm: failed waiting for cluster=\"cluster-candidate-4-21-rh49gm\" in resourcegroup=\"rg-candidate-4-21-6v7gsgmssgvq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\",\n \"name\": \"8ae5c7d2-bbb4-424b-8ab6-07626d56a606\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:59.612142564Z\",\n \"endTime\": \"2026-10-28T15:09:08.641921276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\n correlation ID: 8f132f7f-c309-4e23-a969-712eed0e7d91\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-21-6v7gsgmssgvq/cluster-candidate-4-21-rh49gm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1da958962480\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rh49gm: failed waiting for cluster=\"cluster-candidate-4-21-rh49gm\" in resourcegroup=\"rg-candidate-4-21-6v7gsgmssgvq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\",\n \"name\": \"8ae5c7d2-bbb4-424b-8ab6-07626d56a606\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:59.612142564Z\",\n \"endTime\": \"2026-10-28T15:09:08.641921276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ae5c7d2-bbb4-424b-8ab6-07626d56a606\n correlation ID: 8f132f7f-c309-4e23-a969-712eed0e7d91\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1dbb2c6e90e15fbf133324b328c8affd315fb743792c66a6de75bc9e6d9984bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"8661b303b20422ae894ac47a3a74293388ea0fd48db7bcc2b2dec9182a99758e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-20-5lpmjgvvtgz9/cluster-candidate-4-20-2tt2j5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a4de4ffaa20\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2tt2j5: failed waiting for cluster=\"cluster-candidate-4-20-2tt2j5\" in resourcegroup=\"rg-candidate-4-20-5lpmjgvvtgz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\",\n \"name\": \"be77f741-8538-41d7-b3a5-893532fe7a72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:20.152945198Z\",\n \"endTime\": \"2026-10-28T15:08:28.627516087Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\n correlation ID: 5f3ab66f-143a-4347-ba87-f5cc1c238cc2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-20-5lpmjgvvtgz9/cluster-candidate-4-20-2tt2j5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a4de4ffaa20\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2tt2j5: failed waiting for cluster=\"cluster-candidate-4-20-2tt2j5\" in resourcegroup=\"rg-candidate-4-20-5lpmjgvvtgz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\",\n \"name\": \"be77f741-8538-41d7-b3a5-893532fe7a72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:20.152945198Z\",\n \"endTime\": \"2026-10-28T15:08:28.627516087Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be77f741-8538-41d7-b3a5-893532fe7a72\n correlation ID: 5f3ab66f-143a-4347-ba87-f5cc1c238cc2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"215fed256da158638d4948c1afd16fe347ddd3cd125315b443e5285c13478943","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"667c970b0bc4106596e97478abc3ff3ace5bb451af5139645277b065301185cf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-23-x4xcrxkpwn7j/cluster-candidate-4-23-vkx5ps should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x8d9b5b08c80\u003e: \n failed to create HCP cluster cluster-candidate-4-23-vkx5ps: failed waiting for cluster=\"cluster-candidate-4-23-vkx5ps\" in resourcegroup=\"rg-candidate-4-23-x4xcrxkpwn7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\",\n \"name\": \"195b0201-f716-4d56-87ea-883b4e6144d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:48.141950924Z\",\n \"endTime\": \"2026-10-28T15:08:48.609704416Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\n correlation ID: c7513d16-8aa2-4392-b9e7-c38f94dba9a2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-23-x4xcrxkpwn7j/cluster-candidate-4-23-vkx5ps should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x8d9b5b08c80\u003e: \n failed to create HCP cluster cluster-candidate-4-23-vkx5ps: failed waiting for cluster=\"cluster-candidate-4-23-vkx5ps\" in resourcegroup=\"rg-candidate-4-23-x4xcrxkpwn7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\",\n \"name\": \"195b0201-f716-4d56-87ea-883b4e6144d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:48.141950924Z\",\n \"endTime\": \"2026-10-28T15:08:48.609704416Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/195b0201-f716-4d56-87ea-883b4e6144d4\n correlation ID: c7513d16-8aa2-4392-b9e7-c38f94dba9a2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2897315d20f302eeb23978b18f369c14e1b4a01970df92ab1c1ffe456529ac61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"57a9b4fb81f9a741827a0d6649ef52cee527324f4edfd50bc19545467e6ca18f","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x374e833daa40\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-f4hlsn5n7dp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\",\n \"name\": \"1f756a31-37f0-40e0-82d6-7323816b3a44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:59.191596418Z\",\n \"endTime\": \"2026-10-28T14:12:00.890716747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\n correlation ID: 35fe35ef-43a8-4363-855c-128586bb4a72\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x374e833daa40\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-f4hlsn5n7dp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\",\n \"name\": \"1f756a31-37f0-40e0-82d6-7323816b3a44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:59.191596418Z\",\n \"endTime\": \"2026-10-28T14:12:00.890716747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f756a31-37f0-40e0-82d6-7323816b3a44\n correlation ID: 35fe35ef-43a8-4363-855c-128586bb4a72\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2b4f8fede0f7528798062811860892559c6165917ea17ea8efb1df720569271a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"0c91a1ef66600b81d08ebe7871bd904a4fa12212cb8e7cf459bab1c021971b15","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-wbzs2x at version 4.22.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x105f86c0c5c0\u003e: \n failed to create HCP cluster np-skip-minor-wbzs2x: failed waiting for cluster=\"np-skip-minor-wbzs2x\" in resourcegroup=\"rg-np-skip-minor-wbzs2x-9657t9v84pkg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\",\n \"name\": \"4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:40.618908142Z\",\n \"endTime\": \"2026-10-28T15:50:47.631173688Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, azure-cloud-controller-manager, machine-approver, control-plane-pki-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, kube-storage-version-migrator, catalog-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\n correlation ID: 0acb7fcf-366a-4480-a74c-2c2b50fa0aa0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-wbzs2x at version 4.22.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x105f86c0c5c0\u003e: \n failed to create HCP cluster np-skip-minor-wbzs2x: failed waiting for cluster=\"np-skip-minor-wbzs2x\" in resourcegroup=\"rg-np-skip-minor-wbzs2x-9657t9v84pkg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\",\n \"name\": \"4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:40.618908142Z\",\n \"endTime\": \"2026-10-28T15:50:47.631173688Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, azure-cloud-controller-manager, machine-approver, control-plane-pki-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, kube-storage-version-migrator, catalog-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bd9db08-0d31-47b6-8f6e-287e8e8d4fe4\n correlation ID: 0acb7fcf-366a-4480-a74c-2c2b50fa0aa0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2d2dddf1265e203fe00cd324770d5b92aab9e5bad6df55c6e7db7e08c3b1c6b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"4f8d88d69095d88fc46b514f4dfd5f63a1f808b8d2d8db326a53b245ac6d1c10","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xdfd4e022a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qlszp5g8k9wc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\",\n \"name\": \"06deff3f-928f-4528-ab39-5d28dd6fa99e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:41:22.27269296Z\",\n \"endTime\": \"2026-10-28T15:10:28.476284696Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, openshift-controller-manager, cluster-network-operator, dns-operator, catalog-operator, openshift-route-controller-manager, packageserver, cluster-policy-controller, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\n correlation ID: 09ac3f0b-4c54-44f8-a6e3-2ded2abcc35b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xdfd4e022a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qlszp5g8k9wc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\",\n \"name\": \"06deff3f-928f-4528-ab39-5d28dd6fa99e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:41:22.27269296Z\",\n \"endTime\": \"2026-10-28T15:10:28.476284696Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, openshift-controller-manager, cluster-network-operator, dns-operator, catalog-operator, openshift-route-controller-manager, packageserver, cluster-policy-controller, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/06deff3f-928f-4528-ab39-5d28dd6fa99e\n correlation ID: 09ac3f0b-4c54-44f8-a6e3-2ded2abcc35b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"31ddd0e179bb45a0eaf891f05f622e37d67f093603054a2ac463a6b39b5352bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3e8f1658f5dc1d28d4efd52b26219407f6b0a170d15856d36bbedfd834ddbbe6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-ffl2k6\" with version 4.20.43 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x13122735c680\u003e: \n failed to create HCP cluster cluster-zstream-4-20-ffl2k6: failed waiting for cluster=\"cluster-zstream-4-20-ffl2k6\" in resourcegroup=\"rg-zstream-upgrade-4-20-2qwm89mjrzcr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\",\n \"name\": \"ab266e95-32c5-4fd4-a60d-96a8a32d075c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:33.890129033Z\",\n \"endTime\": \"2026-10-28T15:09:38.467450003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, csi-snapshot-controller-operator, olm-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-collect-profiles, redhat-operators-catalog, machine-approver, kube-controller-manager, hosted-cluster-config-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, cluster-storage-operator, redhat-marketplace-catalog, openshift-apiserver, cluster-version-operator, catalog-operator, kube-scheduler, openshift-controller-manager, dns-operator, cluster-image-registry-operator, packageserver, cluster-autoscaler, certified-operators-catalog, azure-cloud-controller-manager, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\n correlation ID: ba9ee4d9-fd17-4e45-845d-93a236933c1e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-ffl2k6\" with version 4.20.43 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x13122735c680\u003e: \n failed to create HCP cluster cluster-zstream-4-20-ffl2k6: failed waiting for cluster=\"cluster-zstream-4-20-ffl2k6\" in resourcegroup=\"rg-zstream-upgrade-4-20-2qwm89mjrzcr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\",\n \"name\": \"ab266e95-32c5-4fd4-a60d-96a8a32d075c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:33.890129033Z\",\n \"endTime\": \"2026-10-28T15:09:38.467450003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, csi-snapshot-controller-operator, olm-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-collect-profiles, redhat-operators-catalog, machine-approver, kube-controller-manager, hosted-cluster-config-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, cluster-storage-operator, redhat-marketplace-catalog, openshift-apiserver, cluster-version-operator, catalog-operator, kube-scheduler, openshift-controller-manager, dns-operator, cluster-image-registry-operator, packageserver, cluster-autoscaler, certified-operators-catalog, azure-cloud-controller-manager, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ab266e95-32c5-4fd4-a60d-96a8a32d075c\n correlation ID: ba9ee4d9-fd17-4e45-845d-93a236933c1e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3448b403474bcc10587d863fb4408c42aff419d0c2209821f30cf68fc58e1824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"fb1fb5f7ad13ff275d52922aefa84c5111fdc94b3d6726b209d19396eb7f5dbe","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e45224b0c80\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-flztpcd4f9nh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\",\n \"name\": \"56a19993-1d4b-4c82-bc4c-b9cb17083165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:33.142193131Z\",\n \"endTime\": \"2026-10-28T14:12:40.962875758Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\n correlation ID: 6856df15-8a41-4e59-8005-8b021a531d3b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e45224b0c80\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-flztpcd4f9nh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\",\n \"name\": \"56a19993-1d4b-4c82-bc4c-b9cb17083165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:33.142193131Z\",\n \"endTime\": \"2026-10-28T14:12:40.962875758Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/56a19993-1d4b-4c82-bc4c-b9cb17083165\n correlation ID: 6856df15-8a41-4e59-8005-8b021a531d3b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"352a5ddc33b3d4677d0d43bce8c4e8572fcfd75cc007f757e425fb1c3606c1b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"caa3b54dcae0c5626ec06f2239837c5a5dd7fe48e4b5279bc29225d20116ebaa","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2tf7pf\nUnexpected error:\n \u003c*fmt.wrapError | 0x257516f84460\u003e: \n failed to create HCP cluster np-noedge-2tf7pf: failed waiting for cluster=\"np-noedge-2tf7pf\" in resourcegroup=\"rg-np-noedge-2tf7pf-xb8pwtjzh82j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\",\n \"name\": \"92128bbf-b05b-4a42-b590-261b4b996ffc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:31.746407548Z\",\n \"endTime\": \"2026-10-28T15:50:37.764425706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, community-operators-catalog, ingress-operator, csi-snapshot-controller-operator, packageserver, cluster-image-registry-operator, dns-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n correlation ID: b2029c0e-c3d1-46d0-9bac-f3b40a92dc09\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2tf7pf\nUnexpected error:\n \u003c*fmt.wrapError | 0x257516f84460\u003e: \n failed to create HCP cluster np-noedge-2tf7pf: failed waiting for cluster=\"np-noedge-2tf7pf\" in resourcegroup=\"rg-np-noedge-2tf7pf-xb8pwtjzh82j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\",\n \"name\": \"92128bbf-b05b-4a42-b590-261b4b996ffc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:31.746407548Z\",\n \"endTime\": \"2026-10-28T15:50:37.764425706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, community-operators-catalog, ingress-operator, csi-snapshot-controller-operator, packageserver, cluster-image-registry-operator, dns-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n correlation ID: b2029c0e-c3d1-46d0-9bac-f3b40a92dc09\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2be81406fae3edbe2084aa4ec6e8c4bdd917e7ac49e9527c271da2794674dd42","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3b698505c611a4216e75ac2312e1f5de744db1bb0e4b1bb5304f865553bb4918","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"4eacc122e24dff876da529f641ce2f8e88118e1c180fe85bb99ae5977c4eeb02","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-8fz8f5\" with version 4.21.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x17985ea62800\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8fz8f5: failed waiting for cluster=\"cluster-zstream-4-21-8fz8f5\" in resourcegroup=\"rg-zstream-upgrade-4-21-wbqzpvwdc6q6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\",\n \"name\": \"487512e6-43f7-42b5-a694-f4be2585d80c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:20.778612196Z\",\n \"endTime\": \"2026-10-28T15:09:28.776202464Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\n correlation ID: 80cf0427-c640-486c-a738-9d16fdc0c4e5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-8fz8f5\" with version 4.21.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x17985ea62800\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8fz8f5: failed waiting for cluster=\"cluster-zstream-4-21-8fz8f5\" in resourcegroup=\"rg-zstream-upgrade-4-21-wbqzpvwdc6q6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\",\n \"name\": \"487512e6-43f7-42b5-a694-f4be2585d80c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:20.778612196Z\",\n \"endTime\": \"2026-10-28T15:09:28.776202464Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/487512e6-43f7-42b5-a694-f4be2585d80c\n correlation ID: 80cf0427-c640-486c-a738-9d16fdc0c4e5\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3c8a80c9a81ab5994da48ba1233d3004688b6a38bf1e1406b73c7b812ecd5bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3ce674c8fc052d08af13535dc5c626bc93710e8efd03a230ddab4a60c0946118","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x27391d09a3a0\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-vvl2jlrsdzwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\",\n \"name\": \"a5b87783-6a6a-4fee-a31b-7e52a950339f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:22:08.258068936Z\",\n \"endTime\": \"2026-10-28T15:51:17.638485832Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, community-operators-catalog, control-plane-pki-operator, dns-operator, kube-scheduler, cluster-storage-operator, machine-approver, openshift-apiserver, csi-snapshot-controller-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-policy-controller, kube-controller-manager, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, openshift-route-controller-manager, ingress-operator, catalog-operator, packageserver, cluster-autoscaler, openshift-controller-manager, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\n correlation ID: c155e67f-4293-4e6f-9b93-fb56774a919b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x27391d09a3a0\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-vvl2jlrsdzwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\",\n \"name\": \"a5b87783-6a6a-4fee-a31b-7e52a950339f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:22:08.258068936Z\",\n \"endTime\": \"2026-10-28T15:51:17.638485832Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, community-operators-catalog, control-plane-pki-operator, dns-operator, kube-scheduler, cluster-storage-operator, machine-approver, openshift-apiserver, csi-snapshot-controller-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-policy-controller, kube-controller-manager, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, openshift-route-controller-manager, ingress-operator, catalog-operator, packageserver, cluster-autoscaler, openshift-controller-manager, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5b87783-6a6a-4fee-a31b-7e52a950339f\n correlation ID: c155e67f-4293-4e6f-9b93-fb56774a919b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3f9023babe17c69a0bbd07aa4ff48b1c08e3e57e6102ba53853f00ae4c6eed09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"7a2a1051e2c8780e79f7d927cf834797fda62d7186cdfde4d3845eb3ca5413bd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dk6562\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xd48edafd640\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dk6562: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dk6562\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-w6tg4gwbm4zj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\",\n \"name\": \"70464892-f572-4f2a-b4a6-385fdceed9ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:27.209590226Z\",\n \"endTime\": \"2026-10-28T15:09:28.790353292Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\n correlation ID: 5f914fbf-789a-44af-8ded-2b56b051ab07\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dk6562\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xd48edafd640\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dk6562: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dk6562\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-w6tg4gwbm4zj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\",\n \"name\": \"70464892-f572-4f2a-b4a6-385fdceed9ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:27.209590226Z\",\n \"endTime\": \"2026-10-28T15:09:28.790353292Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70464892-f572-4f2a-b4a6-385fdceed9ca\n correlation ID: 5f914fbf-789a-44af-8ded-2b56b051ab07\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"41b140bc7bd75b83918cb966bd50628b91de3a7c7d93c848e078e3f3fd7d3bc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"8c96c1bd310c60c6a4f69d6210e10ae7e4099a5d02be37a4168d23f20c67b47c","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1f0baa544120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1f0baa40b610\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88\",\n },\n \u003c*fmt.wrapErrors | 0x1f0baa5440c0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1f0baa40b610\u003e0x1f0baa40b610 (cyclic reference),\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1f0baa544120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1f0baa40b610\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88\",\n },\n \u003c*fmt.wrapErrors | 0x1f0baa5440c0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1f0baa40b610\u003e0x1f0baa40b610 (cyclic reference),\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded"}},{"row_id":"51200d1db5d0c7f5d58a4067676ea52532c6713b0707b441aa619176a5e9120a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"b02ac677d1d2b2e3d2754f6aaca8c25faaf2ac89e67671af0386ef8433265531","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-qxnh9v\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x9c08c475a0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-qxnh9v: failed waiting for cluster=\"cluster-zstream-4-23-qxnh9v\" in resourcegroup=\"rg-zstream-upgrade-4-23-lzbzhtjskg9m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\",\n \"name\": \"49078e61-6846-4cd0-bfa4-7f73457f1286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:13.468188351Z\",\n \"endTime\": \"2026-10-28T15:09:18.606679709Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-image-registry-operator, olm-operator, kube-scheduler, kube-storage-version-migrator, catalog-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, ingress-operator, openshift-controller-manager, packageserver, konnectivity-agent, hosted-cluster-config-operator, redhat-operators-catalog, dns-operator, control-plane-pki-operator, cluster-policy-controller, cluster-network-operator, kube-controller-manager, cluster-node-tuning-operator, openshift-route-controller-manager, olm-collect-profiles, openshift-apiserver, capi-provider, cluster-storage-operator, csi-snapshot-controller-operator, cluster-version-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\n correlation ID: 0f636737-db3c-44fd-a024-8aa2dae006d5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-qxnh9v\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x9c08c475a0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-qxnh9v: failed waiting for cluster=\"cluster-zstream-4-23-qxnh9v\" in resourcegroup=\"rg-zstream-upgrade-4-23-lzbzhtjskg9m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\",\n \"name\": \"49078e61-6846-4cd0-bfa4-7f73457f1286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:13.468188351Z\",\n \"endTime\": \"2026-10-28T15:09:18.606679709Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-image-registry-operator, olm-operator, kube-scheduler, kube-storage-version-migrator, catalog-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, ingress-operator, openshift-controller-manager, packageserver, konnectivity-agent, hosted-cluster-config-operator, redhat-operators-catalog, dns-operator, control-plane-pki-operator, cluster-policy-controller, cluster-network-operator, kube-controller-manager, cluster-node-tuning-operator, openshift-route-controller-manager, olm-collect-profiles, openshift-apiserver, capi-provider, cluster-storage-operator, csi-snapshot-controller-operator, cluster-version-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49078e61-6846-4cd0-bfa4-7f73457f1286\n correlation ID: 0f636737-db3c-44fd-a024-8aa2dae006d5\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5483f2f5951b0e6936787dd246cbc71fef3b9e05673c532bc58876864e11d43d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"4da04c40a2d7ba226fa1cf6a9bb0dc8d5bdc90ab92882a6f7e685554c77499e5","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x18fb1fe8a1e0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-rwstnfz2pdpd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\",\n \"name\": \"8c849820-891e-4b6d-95ef-663bc731334d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.740667918Z\",\n \"endTime\": \"2026-10-28T15:08:28.665210796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, community-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, cluster-policy-controller, catalog-operator, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, certified-operators-catalog, machine-approver, hosted-cluster-config-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-image-registry-operator, olm-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\n correlation ID: 5c6ea318-da7b-4cf0-897b-e3c0738e611d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x18fb1fe8a1e0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-rwstnfz2pdpd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\",\n \"name\": \"8c849820-891e-4b6d-95ef-663bc731334d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.740667918Z\",\n \"endTime\": \"2026-10-28T15:08:28.665210796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, community-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, cluster-policy-controller, catalog-operator, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, certified-operators-catalog, machine-approver, hosted-cluster-config-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-image-registry-operator, olm-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c849820-891e-4b6d-95ef-663bc731334d\n correlation ID: 5c6ea318-da7b-4cf0-897b-e3c0738e611d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5694a38f6c1b3eaa12928b569ebc4f5f46f6e1031f472e28a5d97932e38d0066","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"da5db98c965dd0440e1b301228d623d359946911d3815f2782f6cad5bd4513c5","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b9a78cc2c00\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-678vggk8hw4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\",\n \"name\": \"95bd7f6e-d28c-4575-8824-c4b761c225fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:43.03617172Z\",\n \"endTime\": \"2026-10-28T15:50:47.66441232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\n correlation ID: 414e22c5-a26a-4c3c-96be-d66c9e15bf90\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b9a78cc2c00\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-678vggk8hw4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\",\n \"name\": \"95bd7f6e-d28c-4575-8824-c4b761c225fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:43.03617172Z\",\n \"endTime\": \"2026-10-28T15:50:47.66441232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/95bd7f6e-d28c-4575-8824-c4b761c225fc\n correlation ID: 414e22c5-a26a-4c3c-96be-d66c9e15bf90\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"59f218702febe97db03aba4d61f43e7354a259eae43fc7859ea379e4138e64c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"f9bb6276ccece58dd6c1b6bfedcca5de455af5123fb87e6b0c63aeb1afa2b571","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x166a5ce4a460\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-xfbmh7tnw8dd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\",\n \"name\": \"e5ff8914-b12e-4cdf-aa40-3986da34c7da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:57.339002209Z\",\n \"endTime\": \"2026-10-28T15:50:57.634345734Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, hosted-cluster-config-operator, azure-cloud-controller-manager, konnectivity-agent, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, dns-operator, certified-operators-catalog, cluster-network-operator, cluster-autoscaler, control-plane-pki-operator, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, community-operators-catalog, machine-approver, kube-controller-manager, kube-scheduler, openshift-controller-manager, ingress-operator, olm-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\n correlation ID: 88b2308c-63cd-42b3-b006-499404a750ad\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x166a5ce4a460\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-xfbmh7tnw8dd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\",\n \"name\": \"e5ff8914-b12e-4cdf-aa40-3986da34c7da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:57.339002209Z\",\n \"endTime\": \"2026-10-28T15:50:57.634345734Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, hosted-cluster-config-operator, azure-cloud-controller-manager, konnectivity-agent, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, dns-operator, certified-operators-catalog, cluster-network-operator, cluster-autoscaler, control-plane-pki-operator, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, community-operators-catalog, machine-approver, kube-controller-manager, kube-scheduler, openshift-controller-manager, ingress-operator, olm-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e5ff8914-b12e-4cdf-aa40-3986da34c7da\n correlation ID: 88b2308c-63cd-42b3-b006-499404a750ad\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"602a41d762bbe47cbb754c6e28d146874f4925a3a82615aa1611a37cc532c8fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"d13b782f0b69e1d4260e04e9bf10320e2c7adb462218158530bd960678b2de8e","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c4df68131a0\u003e: \n failed to create HCP cluster ea-listc5c22j: failed waiting for cluster=\"ea-listc5c22j\" in resourcegroup=\"ea-list-h4pk6vdwpp9q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\",\n \"name\": \"4641515b-36d7-4de2-8c4a-a2ec882ba078\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:41:41.259309811Z\",\n \"endTime\": \"2026-10-28T15:10:48.460536218Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\n correlation ID: 1b213c52-9e35-46b4-84dc-9a0420bced86\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c4df68131a0\u003e: \n failed to create HCP cluster ea-listc5c22j: failed waiting for cluster=\"ea-listc5c22j\" in resourcegroup=\"ea-list-h4pk6vdwpp9q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\",\n \"name\": \"4641515b-36d7-4de2-8c4a-a2ec882ba078\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:41:41.259309811Z\",\n \"endTime\": \"2026-10-28T15:10:48.460536218Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4641515b-36d7-4de2-8c4a-a2ec882ba078\n correlation ID: 1b213c52-9e35-46b4-84dc-9a0420bced86\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"62ca72f675fe4eea40dabf0b3bff4e3f05fa0abb0ff14504f7be9ff120eb047d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"c6fc8da7a6c3fdef78901982a9e72825daf9ba1be7cf33cceedbd6a810b64ca4","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x7c22833c460\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kn67v2kn8ztj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n \"name\": \"d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.28860999Z\",\n \"endTime\": \"2026-10-28T15:50:07.63308505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n correlation ID: de1e5d6e-772f-44e9-aec7-7b1dbc7151a7\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x7c22833c460\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kn67v2kn8ztj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n \"name\": \"d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.28860999Z\",\n \"endTime\": \"2026-10-28T15:50:07.63308505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n correlation ID: de1e5d6e-772f-44e9-aec7-7b1dbc7151a7\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9edc10b3275c8fa4fef403327c8f3071abe3dac4d93e21bb75e84a9f8c3924d6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"67b0bea0da3a20671298ded20a1dfb2c09eee5f1dcc619416687b197721d86f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"b945afde3acdd3156e44fe8bc3986c80cc24618929cebf7f823a1eb5b0391b63","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x375811d74ca0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4dpvzp9p5xb8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\",\n \"name\": \"520cf7c9-acbc-4761-abda-193db7d8b590\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:45:10.07141772Z\",\n \"endTime\": \"2026-10-28T14:14:10.790489596Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\n correlation ID: 1682ab51-6904-4558-9b5e-bab58d966ecb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x375811d74ca0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4dpvzp9p5xb8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\",\n \"name\": \"520cf7c9-acbc-4761-abda-193db7d8b590\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:45:10.07141772Z\",\n \"endTime\": \"2026-10-28T14:14:10.790489596Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/520cf7c9-acbc-4761-abda-193db7d8b590\n correlation ID: 1682ab51-6904-4558-9b5e-bab58d966ecb\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6810f93f8f8c2bea1532886c323e976f7a92b2fc4136c1fe6ed6db7c6914404e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"f6d47638df5c178dc4841bfd12e3a2a7720138c28c5031cd942b4f59a9c78972","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d893d926940\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9zwtss6f6vt9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n \"name\": \"7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:20.447678223Z\",\n \"endTime\": \"2026-10-28T15:08:28.688541915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n correlation ID: 2b84f196-091a-41b2-98d7-60b50644dab4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d893d926940\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9zwtss6f6vt9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n \"name\": \"7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:20.447678223Z\",\n \"endTime\": \"2026-10-28T15:08:28.688541915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n correlation ID: 2b84f196-091a-41b2-98d7-60b50644dab4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18e6120f83ea7fcc84253d55c25e1ca8355971e7accf1adbd2fc6e3196e9463a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"69b168fc6c92cc33f8b97e805e3501daec934ee0ac51a7bf220c5bc92b693cd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3f21188375c2a979f92a3e7d6b3f4e4da0853de8e5e02d474f98123639541247","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-tsmb9zwhf6np/cluster-candidate-5-0-qrqt4m should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x848b1c9a440\u003e: \n failed to create HCP cluster cluster-candidate-5-0-qrqt4m: failed waiting for cluster=\"cluster-candidate-5-0-qrqt4m\" in resourcegroup=\"rg-candidate-5-0-tsmb9zwhf6np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\",\n \"name\": \"ff4d1e16-4c98-4cf9-acfe-6d00538378fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:51.38832954Z\",\n \"endTime\": \"2026-10-28T15:08:58.636801847Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\n correlation ID: 9841ce5e-79da-4501-8301-7fd87b492c29\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-tsmb9zwhf6np/cluster-candidate-5-0-qrqt4m should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x848b1c9a440\u003e: \n failed to create HCP cluster cluster-candidate-5-0-qrqt4m: failed waiting for cluster=\"cluster-candidate-5-0-qrqt4m\" in resourcegroup=\"rg-candidate-5-0-tsmb9zwhf6np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\",\n \"name\": \"ff4d1e16-4c98-4cf9-acfe-6d00538378fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:51.38832954Z\",\n \"endTime\": \"2026-10-28T15:08:58.636801847Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff4d1e16-4c98-4cf9-acfe-6d00538378fd\n correlation ID: 9841ce5e-79da-4501-8301-7fd87b492c29\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6b8bc623fc150faf6e8241be113d7c83d6c6edd64feff0ae4d72d9884fc7d09a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"70799d727cd033b80cb0a6b0871ad37f29b0747617d11e063a3b9d994def6556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"8a964c6e955aa1074acee9f587db3b0f87b653b49549c6bf100107183dfd73cf","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x24f1c7fcefa0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bhc75ljrxzvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\",\n \"name\": \"3ee91092-37d4-4687-8899-086b94094407\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:47.010012275Z\",\n \"endTime\": \"2026-10-28T14:11:50.916418144Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n correlation ID: 881a23bb-07d5-4161-bd1d-8f0790ff0f30\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x24f1c7fcefa0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bhc75ljrxzvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\",\n \"name\": \"3ee91092-37d4-4687-8899-086b94094407\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:47.010012275Z\",\n \"endTime\": \"2026-10-28T14:11:50.916418144Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n correlation ID: 881a23bb-07d5-4161-bd1d-8f0790ff0f30\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76dc45411fd236f9753bcb2666c1041e349ff03625e2cad7d80dae3e454d3e1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"008b7fa14234b7fd53ce92c2a67a34bd489ce71e51f2b07003c764af8afab38b","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x22dc0191a120\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-jc4z76tj8b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\",\n \"name\": \"91e4eda4-eb0e-4ff2-b22e-65959d67929a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:05.747830881Z\",\n \"endTime\": \"2026-10-28T14:12:10.776073825Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\n correlation ID: 93c5577b-f0d7-4fff-80a7-689379e11923\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x22dc0191a120\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-jc4z76tj8b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\",\n \"name\": \"91e4eda4-eb0e-4ff2-b22e-65959d67929a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:05.747830881Z\",\n \"endTime\": \"2026-10-28T14:12:10.776073825Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91e4eda4-eb0e-4ff2-b22e-65959d67929a\n correlation ID: 93c5577b-f0d7-4fff-80a7-689379e11923\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7b0852d3db5bd55fef39a7e01539bf33725f3874a0ccc894d1786101a5a676a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3bfe96f1806920438e26c083d745013e6c6dc2bf1dacb1307401c4f7751ba7d9","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x320c1cc145a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z8bmdl4f2djz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n \"name\": \"6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:08.173393078Z\",\n \"endTime\": \"2026-10-28T14:12:10.809270873Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n correlation ID: 446aa419-d2ee-4b05-829a-2ef1a0102b8b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x320c1cc145a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z8bmdl4f2djz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n \"name\": \"6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:08.173393078Z\",\n \"endTime\": \"2026-10-28T14:12:10.809270873Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n correlation ID: 446aa419-d2ee-4b05-829a-2ef1a0102b8b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7dcc6bf5ac76f62d21a00ae42af68dee8bde908f4f5fb3a1016685d68c6f2ec5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"542dcec1259b249fec86bbe3a02ee09976087e16aaa62ec1af2db79abd808885","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-1-c9tfj9kmq2zp/cluster-candidate-5-1-kdsdqq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x158d9065e120\u003e: \n failed to create HCP cluster cluster-candidate-5-1-kdsdqq: failed waiting for cluster=\"cluster-candidate-5-1-kdsdqq\" in resourcegroup=\"rg-candidate-5-1-c9tfj9kmq2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\",\n \"name\": \"82df0ed4-6ad4-45af-825d-883fcc05e886\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:01.853789521Z\",\n \"endTime\": \"2026-10-28T15:09:08.621567068Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\n correlation ID: bd6e85e8-f244-476d-8e1b-637c7a4beac1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-1-c9tfj9kmq2zp/cluster-candidate-5-1-kdsdqq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x158d9065e120\u003e: \n failed to create HCP cluster cluster-candidate-5-1-kdsdqq: failed waiting for cluster=\"cluster-candidate-5-1-kdsdqq\" in resourcegroup=\"rg-candidate-5-1-c9tfj9kmq2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\",\n \"name\": \"82df0ed4-6ad4-45af-825d-883fcc05e886\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:01.853789521Z\",\n \"endTime\": \"2026-10-28T15:09:08.621567068Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/82df0ed4-6ad4-45af-825d-883fcc05e886\n correlation ID: bd6e85e8-f244-476d-8e1b-637c7a4beac1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7e83c17035aa5dde497dcb8c1300e0e5e8bef358c35cdbc10589bed72777e266","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"aaae8a3dd4bb9cf02e846a792df83408b7d48c787c029d118f706904b2dfba5f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8h2wfx with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x10703f7d02c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8h2wfx: failed waiting for cluster=\"np-version-upgrade-cluster-8h2wfx\" in resourcegroup=\"rg-np-version-upgrade-8h2wfx-x8cr8nn2t9dm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\",\n \"name\": \"46025fd0-85a0-4f95-ac05-96400b1afb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:30.261924644Z\",\n \"endTime\": \"2026-10-28T15:50:37.681049101Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\n correlation ID: e0cd0169-4de0-43d8-a6d4-ceaf4b51be28\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8h2wfx with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x10703f7d02c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8h2wfx: failed waiting for cluster=\"np-version-upgrade-cluster-8h2wfx\" in resourcegroup=\"rg-np-version-upgrade-8h2wfx-x8cr8nn2t9dm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\",\n \"name\": \"46025fd0-85a0-4f95-ac05-96400b1afb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:30.261924644Z\",\n \"endTime\": \"2026-10-28T15:50:37.681049101Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/46025fd0-85a0-4f95-ac05-96400b1afb05\n correlation ID: e0cd0169-4de0-43d8-a6d4-ceaf4b51be28\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8475f90def7bb08dcb744e9491dd9b9ec35d0c44b50b63d3799973caa54ac731","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"17b8a61ee96a7fb815efb7d2c1a5d6db362fee25a96c755eb6f05ec793687672","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x188b1ffab580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pptld2m557n2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\",\n \"name\": \"160dbbc6-24d7-46a4-a575-459e58e52cbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:52.080654002Z\",\n \"endTime\": \"2026-10-28T14:12:00.912767678Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, azure-cloud-controller-manager, dns-operator, cluster-image-registry-operator, packageserver, community-operators-catalog, machine-approver, openshift-route-controller-manager, kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, ingress-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, control-plane-pki-operator, cluster-version-operator, konnectivity-agent, redhat-operators-catalog, capi-provider, cluster-node-tuning-operator, csi-snapshot-controller-operator, olm-operator, redhat-marketplace-catalog, certified-operators-catalog, catalog-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\n correlation ID: b0bdcafa-1d32-4743-b479-c3a60aa49bed\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x188b1ffab580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pptld2m557n2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\",\n \"name\": \"160dbbc6-24d7-46a4-a575-459e58e52cbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:52.080654002Z\",\n \"endTime\": \"2026-10-28T14:12:00.912767678Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, azure-cloud-controller-manager, dns-operator, cluster-image-registry-operator, packageserver, community-operators-catalog, machine-approver, openshift-route-controller-manager, kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, ingress-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, control-plane-pki-operator, cluster-version-operator, konnectivity-agent, redhat-operators-catalog, capi-provider, cluster-node-tuning-operator, csi-snapshot-controller-operator, olm-operator, redhat-marketplace-catalog, certified-operators-catalog, catalog-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/160dbbc6-24d7-46a4-a575-459e58e52cbb\n correlation ID: b0bdcafa-1d32-4743-b479-c3a60aa49bed\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86138cd0ccaf05997f3392aea0ff6272b4916bb34b246754c54cbff87b976737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"194e6484baff053822ee2ec1a0e45066c34368d9f8361a9885d074f172876fe0","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-t7rr5n\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x237ae7f7c240\u003e: \n failed to create HCP cluster cluster-zstream-5-0-t7rr5n: failed waiting for cluster=\"cluster-zstream-5-0-t7rr5n\" in resourcegroup=\"rg-zstream-upgrade-5-0-6fp2zwlhw7bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\",\n \"name\": \"8907c7e5-9e3b-4204-856f-e49835870136\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:08.646687074Z\",\n \"endTime\": \"2026-10-28T15:09:08.445371105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, packageserver, azure-cloud-controller-manager, dns-operator, openshift-apiserver, capi-provider, kube-controller-manager, cluster-policy-controller, cluster-network-operator, certified-operators-catalog, control-plane-pki-operator, openshift-route-controller-manager, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, konnectivity-agent, kube-scheduler, cluster-version-operator, ingress-operator, cluster-autoscaler, redhat-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, olm-operator, machine-approver, kube-storage-version-migrator, hosted-cluster-config-operator, community-operators-catalog, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\n correlation ID: 331db847-c2c2-4f64-8fe5-cddb4a33c4cc\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-t7rr5n\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x237ae7f7c240\u003e: \n failed to create HCP cluster cluster-zstream-5-0-t7rr5n: failed waiting for cluster=\"cluster-zstream-5-0-t7rr5n\" in resourcegroup=\"rg-zstream-upgrade-5-0-6fp2zwlhw7bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\",\n \"name\": \"8907c7e5-9e3b-4204-856f-e49835870136\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:08.646687074Z\",\n \"endTime\": \"2026-10-28T15:09:08.445371105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, packageserver, azure-cloud-controller-manager, dns-operator, openshift-apiserver, capi-provider, kube-controller-manager, cluster-policy-controller, cluster-network-operator, certified-operators-catalog, control-plane-pki-operator, openshift-route-controller-manager, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, konnectivity-agent, kube-scheduler, cluster-version-operator, ingress-operator, cluster-autoscaler, redhat-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, olm-operator, machine-approver, kube-storage-version-migrator, hosted-cluster-config-operator, community-operators-catalog, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8907c7e5-9e3b-4204-856f-e49835870136\n correlation ID: 331db847-c2c2-4f64-8fe5-cddb4a33c4cc\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86b53b30969e0f998e2ffa8b82553969e5575381b3f315f85dbb35ca56caee7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"88fea14a4d2f8221bdce2a4cdad6016802ef185095cab4c049686833df5861ec","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xb05e00c3740\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-xlt7c8hlnpsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\",\n \"name\": \"9b21f4d3-012f-42bb-8427-610849d10abf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:51.039046175Z\",\n \"endTime\": \"2026-10-28T14:12:00.781425402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\n correlation ID: 1ed53069-e0dd-46b7-8d50-006696cb329c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xb05e00c3740\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-xlt7c8hlnpsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\",\n \"name\": \"9b21f4d3-012f-42bb-8427-610849d10abf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:51.039046175Z\",\n \"endTime\": \"2026-10-28T14:12:00.781425402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b21f4d3-012f-42bb-8427-610849d10abf\n correlation ID: 1ed53069-e0dd-46b7-8d50-006696cb329c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"89371c9f017d8ce5925901333aef7fe9ac7854520aaa2f51e0862ee27eaee595","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"2407003efa87938c4c66d5a1180adf130e092f7f7f32eaf09748bc884b75abf6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-28r6sfshkdrv/cluster-candidate-4-22-gcsx6w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xe993535c600\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gcsx6w: failed waiting for cluster=\"cluster-candidate-4-22-gcsx6w\" in resourcegroup=\"rg-candidate-4-22-28r6sfshkdrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\",\n \"name\": \"626ada50-804b-4cf3-9aac-6865d44b51f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:47.572236196Z\",\n \"endTime\": \"2026-10-28T15:08:48.605120634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\n correlation ID: 8c0ce41d-733b-4d14-b437-6022b87a6a82\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-28r6sfshkdrv/cluster-candidate-4-22-gcsx6w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xe993535c600\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gcsx6w: failed waiting for cluster=\"cluster-candidate-4-22-gcsx6w\" in resourcegroup=\"rg-candidate-4-22-28r6sfshkdrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\",\n \"name\": \"626ada50-804b-4cf3-9aac-6865d44b51f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:47.572236196Z\",\n \"endTime\": \"2026-10-28T15:08:48.605120634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/626ada50-804b-4cf3-9aac-6865d44b51f2\n correlation ID: 8c0ce41d-733b-4d14-b437-6022b87a6a82\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"99c3e16efad7bc6b510aef6f348542bdcf6dfe59221d50af3f0cb5ba7c87a043","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"64a40c08ccfc6a3b9dc40685da0aca81380375f2e59b1d6dbc879b99d641bf1f","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6733e84edc0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dwrfj8hlrbzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\",\n \"name\": \"da5198b1-40ba-4a11-8a73-835a5572147c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:30:47.364921948Z\",\n \"endTime\": \"2026-10-28T14:59:48.562085294Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, community-operators-catalog, kube-controller-manager, control-plane-pki-operator, dns-operator, olm-operator, hosted-cluster-config-operator, catalog-operator, machine-approver, openshift-controller-manager, ingress-operator, konnectivity-agent, kube-storage-version-migrator, redhat-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, azure-cloud-controller-manager, cluster-storage-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\n correlation ID: 9056b230-c338-46fd-8ea6-be84f17e39de\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6733e84edc0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dwrfj8hlrbzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\",\n \"name\": \"da5198b1-40ba-4a11-8a73-835a5572147c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:30:47.364921948Z\",\n \"endTime\": \"2026-10-28T14:59:48.562085294Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, community-operators-catalog, kube-controller-manager, control-plane-pki-operator, dns-operator, olm-operator, hosted-cluster-config-operator, catalog-operator, machine-approver, openshift-controller-manager, ingress-operator, konnectivity-agent, kube-storage-version-migrator, redhat-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, azure-cloud-controller-manager, cluster-storage-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/da5198b1-40ba-4a11-8a73-835a5572147c\n correlation ID: 9056b230-c338-46fd-8ea6-be84f17e39de\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"abe6741a63c08ae0ce68006332c14eea328400f1de1e18b4d4bdae95f0d4941a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"a968cd687e32398c82a203a778a7c7df20f60576fe20d79ea6c3c7b23c10b036","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x359d41718240\u003e: \n failed to create HCP cluster np-dg-minor-9nszrs: failed waiting for cluster=\"np-dg-minor-9nszrs\" in resourcegroup=\"rg-np-dg-minor-9nszrs-cwrbzp2nmgh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\",\n \"name\": \"16ac79cb-f3da-4d2b-9e23-05192317b62b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:43.861263865Z\",\n \"endTime\": \"2026-10-28T15:50:47.674486588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, packageserver, machine-approver, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-network-operator, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, hosted-cluster-config-operator, azure-cloud-controller-manager, kube-storage-version-migrator, ingress-operator, cluster-image-registry-operator, control-plane-pki-operator, catalog-operator, redhat-operators-catalog, dns-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\n correlation ID: 34d201c0-84a7-4c87-ac68-3d8fc5f5cfd8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x359d41718240\u003e: \n failed to create HCP cluster np-dg-minor-9nszrs: failed waiting for cluster=\"np-dg-minor-9nszrs\" in resourcegroup=\"rg-np-dg-minor-9nszrs-cwrbzp2nmgh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\",\n \"name\": \"16ac79cb-f3da-4d2b-9e23-05192317b62b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:43.861263865Z\",\n \"endTime\": \"2026-10-28T15:50:47.674486588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, packageserver, machine-approver, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-network-operator, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, hosted-cluster-config-operator, azure-cloud-controller-manager, kube-storage-version-migrator, ingress-operator, cluster-image-registry-operator, control-plane-pki-operator, catalog-operator, redhat-operators-catalog, dns-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16ac79cb-f3da-4d2b-9e23-05192317b62b\n correlation ID: 34d201c0-84a7-4c87-ac68-3d8fc5f5cfd8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aecbe44100aa1d168e270e130fb73252054023dbe6ef18c6191c6db9ba59aac4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3d5e4237e6dcfddccbad19ae692ac75d01ca62ee7e2d4940846201edb7aa7425","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-p5rwk6\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x18410323d680\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-p5rwk6: failed waiting for cluster=\"cp-ystream-upgrade-4-23-p5rwk6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-89hkwz6drhgn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\",\n \"name\": \"c2309fcd-9ee5-4824-b941-841a87860930\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:40.163664046Z\",\n \"endTime\": \"2026-10-28T15:09:48.585939528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\n correlation ID: 7ec55f9b-2f4a-4e1d-8780-4867568c3427\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-p5rwk6\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x18410323d680\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-p5rwk6: failed waiting for cluster=\"cp-ystream-upgrade-4-23-p5rwk6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-89hkwz6drhgn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\",\n \"name\": \"c2309fcd-9ee5-4824-b941-841a87860930\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:40.163664046Z\",\n \"endTime\": \"2026-10-28T15:09:48.585939528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c2309fcd-9ee5-4824-b941-841a87860930\n correlation ID: 7ec55f9b-2f4a-4e1d-8780-4867568c3427\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"af2fa2efa0c5900e816fd5bf94147765ea84f2c3ff833c17d5dd94634127df3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"8b0c818dceeff02dbaca5d871463efcf6a41f805938018d938f9ad5a462f8dce","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c07dd926a00\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-c8lbmc69l4g4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\",\n \"name\": \"0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:39.516281954Z\",\n \"endTime\": \"2026-10-28T14:11:40.788662196Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-network-operator, konnectivity-agent, openshift-route-controller-manager, dns-operator, packageserver, catalog-operator, redhat-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, machine-approver, cluster-node-tuning-operator, ingress-operator, openshift-controller-manager, cluster-version-operator, cluster-image-registry-operator, olm-operator, cluster-policy-controller, certified-operators-catalog, kube-storage-version-migrator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\n correlation ID: a3f2063d-795c-43da-9375-df07755bc07d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c07dd926a00\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-c8lbmc69l4g4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\",\n \"name\": \"0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:39.516281954Z\",\n \"endTime\": \"2026-10-28T14:11:40.788662196Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-network-operator, konnectivity-agent, openshift-route-controller-manager, dns-operator, packageserver, catalog-operator, redhat-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, machine-approver, cluster-node-tuning-operator, ingress-operator, openshift-controller-manager, cluster-version-operator, cluster-image-registry-operator, olm-operator, cluster-policy-controller, certified-operators-catalog, kube-storage-version-migrator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0e1488e0-58a2-4e7f-8b4c-81e77c0439c5\n correlation ID: a3f2063d-795c-43da-9375-df07755bc07d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"af61fc1db0e3dcab41e82a34cb9b89fe53cdd287e7ed8854439e2d81d3f9db31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"94c0703b782d06837b72402fee70c4cd576d6ab0dd07de6e02fc57033e76a4a9","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-dxn5f9\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x11aab681540\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-dxn5f9: failed waiting for cluster=\"cp-ystream-upgrade-5-1-dxn5f9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-v52ld4h7f9q6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\",\n \"name\": \"1ee401ab-408e-486f-9c14-dc1be569aeb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:36.496002206Z\",\n \"endTime\": \"2026-10-28T15:09:38.615736159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\n correlation ID: 41d439a4-d8ea-4b66-a4fa-cd570ec527dc\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-dxn5f9\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x11aab681540\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-dxn5f9: failed waiting for cluster=\"cp-ystream-upgrade-5-1-dxn5f9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-v52ld4h7f9q6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\",\n \"name\": \"1ee401ab-408e-486f-9c14-dc1be569aeb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:36.496002206Z\",\n \"endTime\": \"2026-10-28T15:09:38.615736159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ee401ab-408e-486f-9c14-dc1be569aeb4\n correlation ID: 41d439a4-d8ea-4b66-a4fa-cd570ec527dc\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b1439005f21c2ad9a8923a86dbcf2ea6afb33ca02b1e063a920ade3491b994a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"7ba55d85c0848e7e7fe424c701434aa2745b7d5574cad1964a7875eac6fadb52","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33d30568cba0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rn9mbdwr669k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\",\n \"name\": \"30b1eeed-8a58-491f-a679-903582bd2428\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.381277659Z\",\n \"endTime\": \"2026-10-28T15:50:07.62996819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, kube-scheduler, dns-operator, certified-operators-catalog, openshift-controller-manager, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, machine-approver, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, cluster-autoscaler, control-plane-pki-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n correlation ID: 375547a6-70a5-4d64-8ca8-71de42a27606\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33d30568cba0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rn9mbdwr669k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\",\n \"name\": \"30b1eeed-8a58-491f-a679-903582bd2428\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.381277659Z\",\n \"endTime\": \"2026-10-28T15:50:07.62996819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, kube-scheduler, dns-operator, certified-operators-catalog, openshift-controller-manager, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, machine-approver, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, cluster-autoscaler, control-plane-pki-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n correlation ID: 375547a6-70a5-4d64-8ca8-71de42a27606\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b35e23b4b86f30065ef780fad1cc0171890b104fbd557662651fc1c73d0d5a9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"964c8299f8415fec5f061be2c91ce28301f9952572870c4aee2bce32e9c6371e","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x30c5485c87e0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\",\n \"name\": \"4b718d49-0561-4617-ab78-21f445a937c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:47.79023892Z\",\n \"endTime\": \"2026-10-28T14:12:50.828658903Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\n correlation ID: 14e2f634-75d0-4322-8234-7441ccabc009\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x30c5485c87e0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\",\n \"name\": \"4b718d49-0561-4617-ab78-21f445a937c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:43:47.79023892Z\",\n \"endTime\": \"2026-10-28T14:12:50.828658903Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b718d49-0561-4617-ab78-21f445a937c3\n correlation ID: 14e2f634-75d0-4322-8234-7441ccabc009\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bf63dd1f1c928bb1bef73a0e4b90740abd93177cb10003caa141e6f0f1d7540d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"7c952f7c533ba2ac25282224d93062045cb98f8a5d773ed2839acc1711d4e2eb","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d84c606700\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-gq7pn8p46nb6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\",\n \"name\": \"8792581c-3981-4bcf-b8a1-ffd33680ce15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:50.808216413Z\",\n \"endTime\": \"2026-10-28T14:11:50.825999726Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 24m17s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\n correlation ID: 331b0b35-9b61-4406-b1d0-4fe22089cb00\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d84c606700\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-gq7pn8p46nb6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\",\n \"name\": \"8792581c-3981-4bcf-b8a1-ffd33680ce15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:50.808216413Z\",\n \"endTime\": \"2026-10-28T14:11:50.825999726Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 24m17s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8792581c-3981-4bcf-b8a1-ffd33680ce15\n correlation ID: 331b0b35-9b61-4406-b1d0-4fe22089cb00\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c12729f29c0560fb4de0ce5eee8eea5ca3f6fd0dc34a011cf737b0d236a0e3fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"5bb392afcae0a94809f7d0c4935691ce0cb5e3f427769bf81d509c7da81d71c1","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t67p9t with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe0fc68500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-t67p9t: failed waiting for cluster=\"np-version-upgrade-cluster-t67p9t\" in resourcegroup=\"rg-np-version-upgrade-t67p9t-kpsvm25jmqsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\",\n \"name\": \"9e35dee8-3994-4ba7-9234-74a03d742751\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:15.512939467Z\",\n \"endTime\": \"2026-10-28T15:50:17.639783703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n correlation ID: 8d785f53-f7f0-40a1-aa91-751064d7e050\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t67p9t with version 4.21.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe0fc68500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-t67p9t: failed waiting for cluster=\"np-version-upgrade-cluster-t67p9t\" in resourcegroup=\"rg-np-version-upgrade-t67p9t-kpsvm25jmqsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\",\n \"name\": \"9e35dee8-3994-4ba7-9234-74a03d742751\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:15.512939467Z\",\n \"endTime\": \"2026-10-28T15:50:17.639783703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n correlation ID: 8d785f53-f7f0-40a1-aa91-751064d7e050\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c865da9697b065c65ecdb92ce4c67015c82fe6e7131175dc987f2173ea1459d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"29460d66c06c7032b48f41171cf575e427d2982eda0edd5c652064b13b4a3a96","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bc38a476940\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-78bq6pzzxwrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\",\n \"name\": \"70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:57.38659057Z\",\n \"endTime\": \"2026-10-28T14:12:00.88820442Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, capi-provider, openshift-apiserver, konnectivity-agent, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, packageserver, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-marketplace-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, catalog-operator, community-operators-catalog, certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, kube-controller-manager, kube-scheduler, redhat-operators-catalog, cluster-policy-controller, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\n correlation ID: f60a131e-81c7-4e9f-9c27-ffb965259d3b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bc38a476940\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-78bq6pzzxwrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\",\n \"name\": \"70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:57.38659057Z\",\n \"endTime\": \"2026-10-28T14:12:00.88820442Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, capi-provider, openshift-apiserver, konnectivity-agent, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, packageserver, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-marketplace-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, catalog-operator, community-operators-catalog, certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, kube-controller-manager, kube-scheduler, redhat-operators-catalog, cluster-policy-controller, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/70aa6c6b-01ba-49fe-9aa7-e8f005a8163f\n correlation ID: f60a131e-81c7-4e9f-9c27-ffb965259d3b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cae409acb21fbaba8f01881489c64cda9e3eb86f2fee8cb2787314a3f59eaf8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"91d11f218a6452991755580657a8dab16f1e57a19c912b53be3b78934bbb305d","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xaf67079a580\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lj5gnxtczbk2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\",\n \"name\": \"58216a84-2005-4cee-8574-3ec06d64a912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:46.725416589Z\",\n \"endTime\": \"2026-10-28T14:11:50.908838257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\n correlation ID: da6466cb-9c46-4c0f-a2f1-842af28d09e4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xaf67079a580\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lj5gnxtczbk2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\",\n \"name\": \"58216a84-2005-4cee-8574-3ec06d64a912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:46.725416589Z\",\n \"endTime\": \"2026-10-28T14:11:50.908838257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/58216a84-2005-4cee-8574-3ec06d64a912\n correlation ID: da6466cb-9c46-4c0f-a2f1-842af28d09e4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d823f9ac1c3b3a40a9ab02e7faa1bb04cb66df29de365749ec1d3c39950aec14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"a4eb8e9210102fc87c30ce72625cc97530479383ab1754550319fd1e35820302","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x22e10818450\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-992c2j7mq2x8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-992c2j7mq2x8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x22e1036b8e0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x22e10818450\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-992c2j7mq2x8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-992c2j7mq2x8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x22e1036b8e0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded"}},{"row_id":"d8dffef58a5d6689e071a29d606dd2a5bce622e9c2767faae82b4c85f9e4e485","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"a7147f358a6b4c690915b8e784595bbb5c2a8d551bf2578e9e4fcc175434ec22","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x34041508ad40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v7v8vtjjfpl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\",\n \"name\": \"607a11e6-eef4-46b1-8d10-f3860ca879cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.704531592Z\",\n \"endTime\": \"2026-10-28T14:12:00.891835837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\n correlation ID: 0319cb8e-5768-4bf6-ad23-11bc5967a1e8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x34041508ad40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v7v8vtjjfpl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\",\n \"name\": \"607a11e6-eef4-46b1-8d10-f3860ca879cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.704531592Z\",\n \"endTime\": \"2026-10-28T14:12:00.891835837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/607a11e6-eef4-46b1-8d10-f3860ca879cc\n correlation ID: 0319cb8e-5768-4bf6-ad23-11bc5967a1e8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e0b53ef74188095d0adcda4ab86413487fdd3c905a0be9d8760c5d39d8da14aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"9301f072fd6159abe387e800a4ec1b29d7b4df14329e35241bd16b50ac0a1265","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x38e84538f300\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gzwwf4ztwf95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\",\n \"name\": \"562f9db4-797b-4ac2-824a-b033e3c72f0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:46:50.32793309Z\",\n \"endTime\": \"2026-10-28T14:15:50.815269703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\n correlation ID: a06e94a4-6ab6-4185-9bca-5fc095b3763c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x38e84538f300\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gzwwf4ztwf95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\",\n \"name\": \"562f9db4-797b-4ac2-824a-b033e3c72f0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:46:50.32793309Z\",\n \"endTime\": \"2026-10-28T14:15:50.815269703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/562f9db4-797b-4ac2-824a-b033e3c72f0c\n correlation ID: a06e94a4-6ab6-4185-9bca-5fc095b3763c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e2a5c2f1eda65aba0b8c38347989da33609350b100c19044e47fe61a49352593","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"3b86e0ddfc2f3e17f4d42f49193fb43ee83d3b75659473c5f023382f9499d4b4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x32cee6f94b40\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gkhf8l89gfpt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n \"name\": \"e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.663998842Z\",\n \"endTime\": \"2026-10-28T15:08:28.451578554Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, redhat-operators-catalog, cluster-network-operator, openshift-route-controller-manager, dns-operator, packageserver, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, olm-operator, cluster-version-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n correlation ID: 9764c911-c5b8-4802-ae2b-3bbd27636314\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x32cee6f94b40\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gkhf8l89gfpt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n \"name\": \"e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.663998842Z\",\n \"endTime\": \"2026-10-28T15:08:28.451578554Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, redhat-operators-catalog, cluster-network-operator, openshift-route-controller-manager, dns-operator, packageserver, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, olm-operator, cluster-version-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n correlation ID: 9764c911-c5b8-4802-ae2b-3bbd27636314\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e4461c3bb4bcaa88790896dd5a042bb409a9a8f9593cb52caf6b319f026ca906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"50872cc62b458472eb61bba5961cd6a3739cefd259772bc8671550bbb691403b","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xd2fa34b2580\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-8rnw5kdb6p2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\",\n \"name\": \"e56720ec-9cdb-4896-bffe-b107388330fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:26.535304826Z\",\n \"endTime\": \"2026-10-28T15:50:27.739063015Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, cluster-image-registry-operator, cluster-autoscaler, hosted-cluster-config-operator, cluster-network-operator, dns-operator, catalog-operator, community-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, ingress-operator, kube-controller-manager, cluster-storage-operator, konnectivity-agent, packageserver, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, olm-collect-profiles, csi-snapshot-controller-operator, redhat-operators-catalog, olm-operator, redhat-marketplace-catalog, machine-approver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n correlation ID: 87508886-196b-41dd-b14a-e423d6c69bea\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xd2fa34b2580\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-8rnw5kdb6p2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\",\n \"name\": \"e56720ec-9cdb-4896-bffe-b107388330fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:26.535304826Z\",\n \"endTime\": \"2026-10-28T15:50:27.739063015Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, cluster-image-registry-operator, cluster-autoscaler, hosted-cluster-config-operator, cluster-network-operator, dns-operator, catalog-operator, community-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, ingress-operator, kube-controller-manager, cluster-storage-operator, konnectivity-agent, packageserver, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, olm-collect-profiles, csi-snapshot-controller-operator, redhat-operators-catalog, olm-operator, redhat-marketplace-catalog, machine-approver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n correlation ID: 87508886-196b-41dd-b14a-e423d6c69bea\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e5c4b4328e678a224be888296151b886a3fe5ec107b659838f8ad561b54e9544","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"f0d67bf88f0b1e061ef2f75989731d2852c725ccd7d7653a9aedb9b57213d67e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xgw4k6 with version 4.20.44\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f2f6cd22960\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-xgw4k6: failed waiting for cluster=\"np-version-upgrade-cluster-xgw4k6\" in resourcegroup=\"rg-np-version-upgrade-xgw4k6-fb2pqq2pp9pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\",\n \"name\": \"67af219e-4e40-4a49-ae77-1d482039a909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:37.478622401Z\",\n \"endTime\": \"2026-10-28T15:50:37.723667825Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, kube-scheduler, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n correlation ID: b05be546-d520-46a0-8713-ceb437dfacc8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xgw4k6 with version 4.20.44\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f2f6cd22960\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-xgw4k6: failed waiting for cluster=\"np-version-upgrade-cluster-xgw4k6\" in resourcegroup=\"rg-np-version-upgrade-xgw4k6-fb2pqq2pp9pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\",\n \"name\": \"67af219e-4e40-4a49-ae77-1d482039a909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:37.478622401Z\",\n \"endTime\": \"2026-10-28T15:50:37.723667825Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, kube-scheduler, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n correlation ID: b05be546-d520-46a0-8713-ceb437dfacc8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1865bb63dfef83736260561a6034bb4b023ad00e9cb60922026ce37a8215cdee","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e926e7ac1a4e6ba162f201fdf5dd7858bd241f0fc516e70afeca4efacdce37de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"97e9333a8653308cb25f8af8b95c6bd31e961edee95a0457bde72604aebc1ece","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x10694a205e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tznn7ncp65rs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\",\n \"name\": \"0878d65b-0e30-4ed2-bad3-979640744b0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.463294437Z\",\n \"endTime\": \"2026-10-28T15:50:07.766061264Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, catalog-operator, olm-operator, packageserver, cluster-network-operator, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, openshift-route-controller-manager, dns-operator, certified-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, ingress-operator, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\n correlation ID: fd861975-abd5-431a-85e6-ec6ec36e2032\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x10694a205e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tznn7ncp65rs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\",\n \"name\": \"0878d65b-0e30-4ed2-bad3-979640744b0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.463294437Z\",\n \"endTime\": \"2026-10-28T15:50:07.766061264Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, catalog-operator, olm-operator, packageserver, cluster-network-operator, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, openshift-route-controller-manager, dns-operator, certified-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, ingress-operator, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0878d65b-0e30-4ed2-bad3-979640744b0c\n correlation ID: fd861975-abd5-431a-85e6-ec6ec36e2032\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"efbd68076d5897155b7e0ef23abe9df4c759400a91285e2682c267925c3222d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"b6ebb929d4286b9000cc4edc13f6a4fd80d5b6c19a793ad410789996849c7f23","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b93705ec3c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmcz7jzc8qv4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\",\n \"name\": \"9ed078ac-98dc-4c85-b217-6c8192a338f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:30.728963233Z\",\n \"endTime\": \"2026-10-28T15:08:38.445396808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\n correlation ID: 23b7ed7f-ac0f-446d-a8b8-7064395acb17\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b93705ec3c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmcz7jzc8qv4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\",\n \"name\": \"9ed078ac-98dc-4c85-b217-6c8192a338f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:30.728963233Z\",\n \"endTime\": \"2026-10-28T15:08:38.445396808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ed078ac-98dc-4c85-b217-6c8192a338f1\n correlation ID: 23b7ed7f-ac0f-446d-a8b8-7064395acb17\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f29872d783621fe02969ef8144ab144878be3be8f0c3a2a4c18d0facb6b38c32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"b5c0928fd45bfb1b4de0807a52343c6bd769a668574d92535003b3a4c3fa886a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c5ef3c68380\u003e: \n failed to create HCP cluster np-downgrade-xtqrzt: failed waiting for cluster=\"np-downgrade-xtqrzt\" in resourcegroup=\"rg-np-downgrade-xtqrzt-r62kmdslppc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\",\n \"name\": \"cc133c6d-1ecb-4233-846e-389835238d32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:46.526133596Z\",\n \"endTime\": \"2026-10-28T15:50:47.81681125Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\n correlation ID: e4a4ba53-062c-4085-8daa-67420223bf34\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c5ef3c68380\u003e: \n failed to create HCP cluster np-downgrade-xtqrzt: failed waiting for cluster=\"np-downgrade-xtqrzt\" in resourcegroup=\"rg-np-downgrade-xtqrzt-r62kmdslppc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\",\n \"name\": \"cc133c6d-1ecb-4233-846e-389835238d32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:46.526133596Z\",\n \"endTime\": \"2026-10-28T15:50:47.81681125Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cc133c6d-1ecb-4233-846e-389835238d32\n correlation ID: e4a4ba53-062c-4085-8daa-67420223bf34\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f38926b6f5445d832a8312a40fca4244395452ee8b3eb2d2b270895d5d562417","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"fa11fc83ee685e2b0b8139fe2a6ad2007ed47287825b00289738db00e3401559","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a82fa16ec00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-6pgk2lfq2z5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\",\n \"name\": \"452dde4b-2f11-4bc7-8844-52cf988c2a61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.742575586Z\",\n \"endTime\": \"2026-10-28T14:12:00.885748443Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\n correlation ID: 0ef05e97-5d14-4a27-8e3b-6769d48b8868\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a82fa16ec00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-6pgk2lfq2z5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\",\n \"name\": \"452dde4b-2f11-4bc7-8844-52cf988c2a61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.742575586Z\",\n \"endTime\": \"2026-10-28T14:12:00.885748443Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/452dde4b-2f11-4bc7-8844-52cf988c2a61\n correlation ID: 0ef05e97-5d14-4a27-8e3b-6769d48b8868\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f66c21f031bfa1317cc5b108a56edf236cdafc35aeb48a9fbd473fae759b1345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"088a3092facb9e749cf94ca46a9c078302ab260e25eb1075b25fd77f9f9504a5","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x371784ecc8e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-xk2p7njp5dsf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\",\n \"name\": \"b13f5e9a-eaea-4399-8cfa-fbd539c1236b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.575502401Z\",\n \"endTime\": \"2026-10-28T15:50:07.653220479Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, certified-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-marketplace-catalog, machine-approver, kube-controller-manager, dns-operator, cluster-version-operator, openshift-route-controller-manager, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, azure-cloud-controller-manager, olm-operator, community-operators-catalog, ingress-operator, konnectivity-agent, packageserver, cluster-policy-controller, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\n correlation ID: 5f2b42c0-ed6b-46a2-99e5-52df9f0918c9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x371784ecc8e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-xk2p7njp5dsf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\",\n \"name\": \"b13f5e9a-eaea-4399-8cfa-fbd539c1236b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T15:21:06.575502401Z\",\n \"endTime\": \"2026-10-28T15:50:07.653220479Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, certified-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-marketplace-catalog, machine-approver, kube-controller-manager, dns-operator, cluster-version-operator, openshift-route-controller-manager, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, azure-cloud-controller-manager, olm-operator, community-operators-catalog, ingress-operator, konnectivity-agent, packageserver, cluster-policy-controller, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b13f5e9a-eaea-4399-8cfa-fbd539c1236b\n correlation ID: 5f2b42c0-ed6b-46a2-99e5-52df9f0918c9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f8289cb68fc2ec008c2034a4aca3ff43774c75dc2a822e9e7a87f8f59b55010a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"f16235cd783938327ef4a9a477b7e7b8c1f5b1782a0f3686b7101daa1e644956","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f31a8d27a80\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fr6kmzk5bhwz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\",\n \"name\": \"bf5eb7a7-3ba7-4a13-a695-d60814412b79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.737045251Z\",\n \"endTime\": \"2026-10-28T15:08:28.661748592Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\n correlation ID: 27846e27-fa3f-4aac-9b56-7baea8b38233\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f31a8d27a80\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fr6kmzk5bhwz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\",\n \"name\": \"bf5eb7a7-3ba7-4a13-a695-d60814412b79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:39:19.737045251Z\",\n \"endTime\": \"2026-10-28T15:08:28.661748592Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf5eb7a7-3ba7-4a13-a695-d60814412b79\n correlation ID: 27846e27-fa3f-4aac-9b56-7baea8b38233\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fc630c90f33a191d0fc21a597afde3a532b23ce70f7252ebb55b224e44c957cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"6ecbcab48c36fbfa7e11bc04e865a63921a9d07e34834d7625911e0fc3cce406","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bdgcnw\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x10183589cbc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bdgcnw: failed waiting for cluster=\"cp-ystream-upgrade-5-0-bdgcnw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cmbc75g7mbtn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\",\n \"name\": \"71951328-f6e7-4428-bfbf-e48b2270b472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:58.169629589Z\",\n \"endTime\": \"2026-10-28T15:09:58.460283392Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\n correlation ID: 49d0f624-19e9-4c7a-9bf0-8e4f0d5a48ea\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bdgcnw\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x10183589cbc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bdgcnw: failed waiting for cluster=\"cp-ystream-upgrade-5-0-bdgcnw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cmbc75g7mbtn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\",\n \"name\": \"71951328-f6e7-4428-bfbf-e48b2270b472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T14:40:58.169629589Z\",\n \"endTime\": \"2026-10-28T15:09:58.460283392Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/71951328-f6e7-4428-bfbf-e48b2270b472\n correlation ID: 49d0f624-19e9-4c7a-9bf0-8e4f0d5a48ea\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ff3e675ddd8976fa9c64f0e94b71f889688c0ba296e8e2bec56a54ed21d6810b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","failed_at":"e2e","signature_id":"234ea90fae8d4c8e76a6d720fc3746417508af7ee1d3235413e8209300f96430","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x186bf96476a0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f9vz9brpqmbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n \"name\": \"07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.996297525Z\",\n \"endTime\": \"2026-10-28T14:12:00.914226401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n correlation ID: b4a6856e-5332-498a-8060-89b9ebb7175d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x186bf96476a0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f9vz9brpqmbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n \"name\": \"07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T13:42:56.996297525Z\",\n \"endTime\": \"2026-10-28T14:12:00.914226401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n correlation ID: b4a6856e-5332-498a-8060-89b9ebb7175d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cbf83efdfcbd15587be949946e10b03eab123e752a2d391c330556621de8f835","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent, kube-controller-manager, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":66,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1865bb63dfef83736260561a6034bb4b023ad00e9cb60922026ce37a8215cdee","18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","18e6120f83ea7fcc84253d55c25e1ca8355971e7accf1adbd2fc6e3196e9463a","1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","2be81406fae3edbe2084aa4ec6e8c4bdd917e7ac49e9527c271da2794674dd42","4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","9edc10b3275c8fa4fef403327c8f3071abe3dac4d93e21bb75e84a9f8c3924d6","9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","cbf83efdfcbd15587be949946e10b03eab123e752a2d391c330556621de8f835","d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c"],"matched_occurrences":66,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2115262480262369280","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-28T02:00:49Z","started_at":"2026-10-28T02:00:49Z","completed_at":"2026-10-28T03:59:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:11:14.174490695Z","timing_metadata_checked_at":"2026-10-28T04:11:14.174490695Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:11:14.178392083Z","region_metadata_checked_at":"2026-10-28T04:11:14.178392083Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"457d7eddbcfa6211a80df0d3c9cf4ec98fc1ba14f337876267623f56fa066ae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2115262480262369280","occurred_at":"2026-10-28T02:00:49Z","failed_at":"e2e","signature_id":"f37db33d97ba0fa58d50483c591fa7b5d1cded288f97755f3eda96c7fd078fd8","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12961691c220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjbhtbfkbk4z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n \"name\": \"b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T02:05:53.148166937Z\",\n \"endTime\": \"2026-10-28T02:34:59.674611282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n correlation ID: 5a9d9ef9-c6e1-45c7-a60d-afc7b67a11c1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12961691c220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjbhtbfkbk4z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n \"name\": \"b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-28T02:05:53.148166937Z\",\n \"endTime\": \"2026-10-28T02:34:59.674611282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n correlation ID: 5a9d9ef9-c6e1-45c7-a60d-afc7b67a11c1\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d6ef0f747748f3897783008b2ad9c103a4591a86a275b0ca77bdd29c34e6b0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0d6ef0f747748f3897783008b2ad9c103a4591a86a275b0ca77bdd29c34e6b0f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115253837576015872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T01:26:29Z","started_at":"2026-10-28T01:26:29Z","completed_at":"2026-10-28T03:49:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:01:06.053613894Z","timing_metadata_checked_at":"2026-10-28T04:01:06.053613894Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:01:06.057760881Z","region_metadata_checked_at":"2026-10-28T04:01:06.057760881Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115253284011773952","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T01:24:17Z","started_at":"2026-10-28T01:24:17Z","completed_at":"2026-10-28T03:58:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T04:11:14.079310696Z","timing_metadata_checked_at":"2026-10-28T04:11:14.079310696Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T04:11:14.082790685Z","region_metadata_checked_at":"2026-10-28T04:11:14.082790685Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115251569904586752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-28T01:17:28Z","started_at":"2026-10-28T01:17:28Z","completed_at":"2026-10-28T03:34:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T03:50:06.223393858Z","timing_metadata_checked_at":"2026-10-28T03:50:06.223393858Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T03:50:06.226494149Z","region_metadata_checked_at":"2026-10-28T03:50:06.226494149Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}