{"meta":{"date":"2026-10-20","timezone":"UTC","generated_at":"2026-10-29T01:44:57Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112690524891648000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-20T23:40:47Z","started_at":"2026-10-20T23:40:47Z","completed_at":"2026-10-21T02:19:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T02:35:43.291901712Z","timing_metadata_checked_at":"2026-10-21T02:35:43.291901712Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T02:35:43.296096395Z","region_metadata_checked_at":"2026-10-21T02:35:43.296096395Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112689080448847872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-20T23:35:03Z","started_at":"2026-10-20T23:35:03Z","completed_at":"2026-10-21T01:53:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T02:04:31.008155334Z","timing_metadata_checked_at":"2026-10-21T02:04:31.008155334Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T02:04:31.011420721Z","region_metadata_checked_at":"2026-10-21T02:04:31.011420721Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112687431340789760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-20T23:28:30Z","started_at":"2026-10-20T23:28:30Z","completed_at":"2026-10-21T01:58:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T02:14:45.136580956Z","timing_metadata_checked_at":"2026-10-21T02:14:45.136580956Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T02:14:45.142460248Z","region_metadata_checked_at":"2026-10-21T02:14:45.142460248Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-20T23:00:09Z","started_at":"2026-10-20T23:00:09Z","completed_at":"2026-10-21T00:44:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T00:51:42.58052649Z","timing_metadata_checked_at":"2026-10-21T00:51:42.58052649Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T00:51:42.583179485Z","region_metadata_checked_at":"2026-10-21T00:51:42.583179485Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1bd11e4b51d161c27fbc6b92e3068d42bff2c062d9d30704c545f40b3b889d47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","failed_at":"e2e","signature_id":"94536779431dbad340de6d98c947080d665396d79fa2590db0ca47b199b6f768","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x32eaa59cc210\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 37d0ceb3-cb28-4457-98b9-e7ba4d280731\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x32eaa59cc210\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 37d0ceb3-cb28-4457-98b9-e7ba4d280731\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3a370d9f7cc41523c68a83cbada18c34a8d05d8f5f1db25eb61632a5aad42ab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","failed_at":"e2e","signature_id":"9dc909a5d64b01d035d949a7efc84e5c1444d43ea7fa70c67f943609bf582bd6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x31136f545320\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2ljfl2x47zf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2ljfl2x47zf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 777139e5-6531-49a8-aaf3-2f08e2ba980d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x31136f545320\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2ljfl2x47zf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2ljfl2x47zf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 777139e5-6531-49a8-aaf3-2f08e2ba980d\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8fda17e358c9a978b06d2412e9086206c9dc7405fc48534df73649073d58acb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","failed_at":"e2e","signature_id":"cb24ba12df08b46fc09a4f838a3aa039e729f29b3406c143bedd14694edad6e2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x9ee9817d500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-68bhgsvbbnq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-68bhgsvbbnq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 50029d6c-026d-499e-b5db-c371c234a247\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x9ee9817d500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-68bhgsvbbnq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-68bhgsvbbnq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 50029d6c-026d-499e-b5db-c371c234a247\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9f3ed279b232ae93f7b86e572c6051993a6b69483246bd0d38f0a408f754e609","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","failed_at":"e2e","signature_id":"51afcfdde45a0aa3ec6d3b1b3015660ec4a7a6ce7dbf663ce5a3a178d9ff23ee","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x16962b1f6f00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 9ef8b68a-bcc4-4e41-9cf0-46d6272fce74\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x16962b1f6f00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 9ef8b68a-bcc4-4e41-9cf0-46d6272fce74\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f09762ee29b000e69f6a93976946c5860d9f36318842c6fcd2cc8cf4b71bd23f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","failed_at":"e2e","signature_id":"8ad7e88444f4ffe5bf5faead1e295bf1c4daae12a4cbae9a31a7d7dd1ffe08b2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xaa2ec6a3830\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 11208f71-8029-42d4-aa43-9f639823791f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xaa2ec6a3830\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 11208f71-8029-42d4-aa43-9f639823791f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112638085463281664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-20T20:12:25Z","started_at":"2026-10-20T20:12:25Z","completed_at":"2026-10-20T22:30:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T22:45:41.552484666Z","timing_metadata_checked_at":"2026-10-20T22:45:41.552484666Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T22:45:41.55610986Z","region_metadata_checked_at":"2026-10-20T22:45:41.55610986Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-20T04:23:38Z","started_at":"2026-10-20T04:23:38Z","completed_at":"2026-10-20T07:25:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T07:33:56.582358866Z","timing_metadata_checked_at":"2026-10-20T07:33:56.582358866Z","region":"eastus2euap","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T07:33:56.585057658Z","region_metadata_checked_at":"2026-10-20T07:33:56.585057658Z"},"failed_at":["e2e"],"failed_test_count":61,"occurrences":[{"row_id":"045763d426b3ab518f8c37f7ad0852e162dd43654c3557d03cf2a8cf51ecfa01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"9572b1168dd4e09bd2006ea91b7e7cbda85a1354aa16390899d53973f05cccba","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-shckcq with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x17cd02074520\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-shckcq: failed waiting for cluster=\"np-version-upgrade-cluster-shckcq\" in resourcegroup=\"rg-np-version-upgrade-shckcq-xnqxw8phxcj5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n \"name\": \"33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:29.730721891Z\",\n \"endTime\": \"2026-10-20T06:54:30.236471474Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n correlation ID: 5197a65a-354b-4a54-a076-091fbd77ba87\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-shckcq with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x17cd02074520\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-shckcq: failed waiting for cluster=\"np-version-upgrade-cluster-shckcq\" in resourcegroup=\"rg-np-version-upgrade-shckcq-xnqxw8phxcj5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n \"name\": \"33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:29.730721891Z\",\n \"endTime\": \"2026-10-20T06:54:30.236471474Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n correlation ID: 5197a65a-354b-4a54-a076-091fbd77ba87\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"08b6d0713b4aa0d0507551dc09b71cae3f9851392d19a57a2f4acdb447eade1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"914b343a149d69ce3d4ecefdfb503cd55563fec6ce1e76edecf68478fb484783","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x4253280e700\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-kbzdgx8s4f5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\",\n \"name\": \"c59dc064-358a-41e7-96ea-245df4c3c210\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:20.293860756Z\",\n \"endTime\": \"2026-10-20T05:15:25.385277801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, community-operators-catalog, cluster-network-operator, ingress-operator, cluster-policy-controller, certified-operators-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, olm-operator, packageserver, konnectivity-agent, redhat-marketplace-catalog, openshift-controller-manager, kube-controller-manager, cluster-version-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\n correlation ID: 106721c0-9cae-4777-88ad-d02999d86677\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x4253280e700\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-kbzdgx8s4f5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\",\n \"name\": \"c59dc064-358a-41e7-96ea-245df4c3c210\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:20.293860756Z\",\n \"endTime\": \"2026-10-20T05:15:25.385277801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, community-operators-catalog, cluster-network-operator, ingress-operator, cluster-policy-controller, certified-operators-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, olm-operator, packageserver, konnectivity-agent, redhat-marketplace-catalog, openshift-controller-manager, kube-controller-manager, cluster-version-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c59dc064-358a-41e7-96ea-245df4c3c210\n correlation ID: 106721c0-9cae-4777-88ad-d02999d86677\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"09e4b91caf9dbbd694e896eeb107507ef6d835806745c74bc4e59807fd95d421","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"290c508f7f2e85c3460635eae7ea41f5b7780ce968d6ca80abb77cd3bee95036","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-ld7txs\" with version 4.21.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a897434c160\u003e: \n failed to create HCP cluster cluster-zstream-4-21-ld7txs: failed waiting for cluster=\"cluster-zstream-4-21-ld7txs\" in resourcegroup=\"rg-zstream-upgrade-4-21-nksp586gvs74\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\",\n \"name\": \"108f8007-9c36-4f2e-a6de-449e5b71d4fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:57.651134348Z\",\n \"endTime\": \"2026-10-20T06:04:00.668104484Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, certified-operators-catalog, capi-provider, dns-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, machine-approver, cluster-version-operator, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, cluster-policy-controller, cluster-network-operator, redhat-marketplace-catalog, openshift-apiserver, openshift-route-controller-manager, kube-controller-manager, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\n correlation ID: 9e830a06-ed0a-4cfe-a52d-18fdb5da5647\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-ld7txs\" with version 4.21.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a897434c160\u003e: \n failed to create HCP cluster cluster-zstream-4-21-ld7txs: failed waiting for cluster=\"cluster-zstream-4-21-ld7txs\" in resourcegroup=\"rg-zstream-upgrade-4-21-nksp586gvs74\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\",\n \"name\": \"108f8007-9c36-4f2e-a6de-449e5b71d4fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:57.651134348Z\",\n \"endTime\": \"2026-10-20T06:04:00.668104484Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, certified-operators-catalog, capi-provider, dns-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, machine-approver, cluster-version-operator, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, cluster-policy-controller, cluster-network-operator, redhat-marketplace-catalog, openshift-apiserver, openshift-route-controller-manager, kube-controller-manager, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/108f8007-9c36-4f2e-a6de-449e5b71d4fa\n correlation ID: 9e830a06-ed0a-4cfe-a52d-18fdb5da5647\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0eba1ec5ce02c61e624f36d7b12a08f528513aa599e98c75207790b276ed70b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"010478f08c7ee8fb5fb2617ab302b6ce848de473bb39c93c97f69270deb122a0","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-zjk46l with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0x1de0e4b15080\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zjk46l: failed waiting for cluster=\"np-version-upgrade-cluster-zjk46l\" in resourcegroup=\"rg-np-version-upgrade-zjk46l-52b2swhcrs62\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\",\n \"name\": \"7572214c-58bb-42db-9c41-46711f0d39e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:21:20.599092026Z\",\n \"endTime\": \"2026-10-20T06:50:20.225694377Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, dns-operator, control-plane-pki-operator, kube-scheduler, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, community-operators-catalog, cluster-network-operator, packageserver, kube-controller-manager, openshift-controller-manager, catalog-operator, olm-collect-profiles, capi-provider, olm-operator, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, certified-operators-catalog, machine-approver, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n correlation ID: 2a2f8f2f-1289-48c9-97d5-1f5137565aa0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-zjk46l with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0x1de0e4b15080\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zjk46l: failed waiting for cluster=\"np-version-upgrade-cluster-zjk46l\" in resourcegroup=\"rg-np-version-upgrade-zjk46l-52b2swhcrs62\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\",\n \"name\": \"7572214c-58bb-42db-9c41-46711f0d39e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:21:20.599092026Z\",\n \"endTime\": \"2026-10-20T06:50:20.225694377Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, dns-operator, control-plane-pki-operator, kube-scheduler, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, community-operators-catalog, cluster-network-operator, packageserver, kube-controller-manager, openshift-controller-manager, catalog-operator, olm-collect-profiles, capi-provider, olm-operator, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, certified-operators-catalog, machine-approver, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n correlation ID: 2a2f8f2f-1289-48c9-97d5-1f5137565aa0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"110e8e0d4ddc78c0cf1c0804bfeb8b9bc418e1d03b8f6f53fc3fb5b2c30c6a55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"0c0b1582bdefb57620af76df4308630b0656dcc4f4d034080e6eb8bbcee22919","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1814681e0940\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zplfd4dszpf2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\",\n \"name\": \"bf6362d3-f073-4cb8-94d2-589eedf69b41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:16.816656428Z\",\n \"endTime\": \"2026-10-20T05:15:25.500520178Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\n correlation ID: 7d9c2d2a-11a0-44b1-9220-2383b9391115\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1814681e0940\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zplfd4dszpf2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\",\n \"name\": \"bf6362d3-f073-4cb8-94d2-589eedf69b41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:16.816656428Z\",\n \"endTime\": \"2026-10-20T05:15:25.500520178Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bf6362d3-f073-4cb8-94d2-589eedf69b41\n correlation ID: 7d9c2d2a-11a0-44b1-9220-2383b9391115\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"18b8dde0cf9e31b01bf7c7b596bbd33d11f187146400e11d23146c1f4e97a110","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"f1d59441d9727804d5c781df8711ee40935778aa3e5e87010831b826d8329748","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2072efeace0\u003e: \n failed to create HCP cluster ea-listmlx87m: failed waiting for cluster=\"ea-listmlx87m\" in resourcegroup=\"ea-list-nlprzq99rzmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n \"name\": \"9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:03.327418936Z\",\n \"endTime\": \"2026-10-20T06:06:10.675381294Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 14m23s ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n correlation ID: b09d3f5f-9c93-4d50-8097-63a061652137\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2072efeace0\u003e: \n failed to create HCP cluster ea-listmlx87m: failed waiting for cluster=\"ea-listmlx87m\" in resourcegroup=\"ea-list-nlprzq99rzmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n \"name\": \"9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:03.327418936Z\",\n \"endTime\": \"2026-10-20T06:06:10.675381294Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 14m23s ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n correlation ID: b09d3f5f-9c93-4d50-8097-63a061652137\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18083b0da87c5f910711dbd500417dc67679d796a27ea6259b1d8bd9fe4e29dd","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"19b51d86a155e7870d3736f39decc129b5847f37acbaa996452e961d5764efec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"e9ce23b3d7e9d97edaca0e32d8e720245c2455d9d7bccc974ec7f9ca371f3bf4","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x33c31e5e05a0\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-vfbnvrfhkdfx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\",\n \"name\": \"638672ea-8701-4e7b-9693-06e2cd06c72d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:21:41.650397537Z\",\n \"endTime\": \"2026-10-20T06:50:50.156260547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, olm-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, catalog-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\n correlation ID: 641405cd-c8f3-423a-b597-b3eb8421c184\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x33c31e5e05a0\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-vfbnvrfhkdfx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\",\n \"name\": \"638672ea-8701-4e7b-9693-06e2cd06c72d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:21:41.650397537Z\",\n \"endTime\": \"2026-10-20T06:50:50.156260547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, olm-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, catalog-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/638672ea-8701-4e7b-9693-06e2cd06c72d\n correlation ID: 641405cd-c8f3-423a-b597-b3eb8421c184\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1a155709ae2b51fdf0677a170eaee5f8ad2921bf942028136ca8bf81146a319c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"55d3cab10d4262bc680cdea2d0125d8ba2c43778388a8535e053d3394c2f29ce","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-9sdkkc64jvl9/cluster-candidate-4-22-gsn9h6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c1a027d2a40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gsn9h6: failed waiting for cluster=\"cluster-candidate-4-22-gsn9h6\" in resourcegroup=\"rg-candidate-4-22-9sdkkc64jvl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\",\n \"name\": \"600782b4-0747-40d0-bcd5-912dd340e256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:09.935713063Z\",\n \"endTime\": \"2026-10-20T06:03:10.688594438Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\n correlation ID: e11289d6-4c64-48ad-97eb-59909cb94623\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-9sdkkc64jvl9/cluster-candidate-4-22-gsn9h6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c1a027d2a40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gsn9h6: failed waiting for cluster=\"cluster-candidate-4-22-gsn9h6\" in resourcegroup=\"rg-candidate-4-22-9sdkkc64jvl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\",\n \"name\": \"600782b4-0747-40d0-bcd5-912dd340e256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:09.935713063Z\",\n \"endTime\": \"2026-10-20T06:03:10.688594438Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/600782b4-0747-40d0-bcd5-912dd340e256\n correlation ID: e11289d6-4c64-48ad-97eb-59909cb94623\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1b22abb9bc6f8630726af5ebbc1eb554a491ea022f826836fdcdf2c4c40cd0ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"e2840043f5fd0a2eda7b3ca11b292f1b1f1e365717f405d42a32dcb35f6fd4c6","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22b0bd1ab540\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ggjjtn4mcs96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\",\n \"name\": \"22864d23-2c89-417c-b977-452042988bd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:33.313234981Z\",\n \"endTime\": \"2026-10-20T05:15:35.477948541Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, kube-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-node-tuning-operator, cluster-image-registry-operator, packageserver, dns-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-controller-manager, ingress-operator, certified-operators-catalog, control-plane-pki-operator, cluster-storage-operator, machine-approver, openshift-apiserver, olm-collect-profiles, redhat-marketplace-catalog, cluster-policy-controller, olm-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\n correlation ID: 4573f812-a41d-41ae-bb22-e273d3b837f3\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22b0bd1ab540\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ggjjtn4mcs96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\",\n \"name\": \"22864d23-2c89-417c-b977-452042988bd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:33.313234981Z\",\n \"endTime\": \"2026-10-20T05:15:35.477948541Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, kube-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-node-tuning-operator, cluster-image-registry-operator, packageserver, dns-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-controller-manager, ingress-operator, certified-operators-catalog, control-plane-pki-operator, cluster-storage-operator, machine-approver, openshift-apiserver, olm-collect-profiles, redhat-marketplace-catalog, cluster-policy-controller, olm-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/22864d23-2c89-417c-b977-452042988bd7\n correlation ID: 4573f812-a41d-41ae-bb22-e273d3b837f3\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1fb208ed8ebdb05bf5ca1e2cd98c39d4aa93da1f1a4962eb2d9c07f5e4af0f85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"928184f02b9d110413b220f12fab11223a87fd1749e66886968a83d4f750b438","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x125f6577ccc0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-wfwdpzxn5ttj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\",\n \"name\": \"4261d901-919c-4b7a-88a4-6f902ead948c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:38:00.472569744Z\",\n \"endTime\": \"2026-10-20T06:07:00.66711018Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\n correlation ID: b23df162-1408-473f-b0de-20a6b3f91a5c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x125f6577ccc0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-wfwdpzxn5ttj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\",\n \"name\": \"4261d901-919c-4b7a-88a4-6f902ead948c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:38:00.472569744Z\",\n \"endTime\": \"2026-10-20T06:07:00.66711018Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4261d901-919c-4b7a-88a4-6f902ead948c\n correlation ID: b23df162-1408-473f-b0de-20a6b3f91a5c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1fe0e4d0dc98a977319fc7c48b5eb07de1af920e428a30b85466af5af478a168","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"71922cf41f134281ebadb3bdb4fb82e42493d9860264a253b22bc5350e92e1c2","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x253bb877e5a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nng4k2kf29bc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\",\n \"name\": \"b46a45a4-dcba-413a-9b74-f904cf76e493\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:17.820195399Z\",\n \"endTime\": \"2026-10-20T06:58:20.185763239Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n correlation ID: a247ec97-7f49-437d-bb0b-6d5fc996e055\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x253bb877e5a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nng4k2kf29bc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\",\n \"name\": \"b46a45a4-dcba-413a-9b74-f904cf76e493\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:17.820195399Z\",\n \"endTime\": \"2026-10-20T06:58:20.185763239Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n correlation ID: a247ec97-7f49-437d-bb0b-6d5fc996e055\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2720dd6709eebfb5f12cea9766ab740d431b9b242de609a743e55b2b9db18af2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"0f0d911e3baacda1fb2ae94c5502b51bfe46c06febf46f973e63beec4d9e1ee8","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x32189916ae0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-p4k6hc7mknvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\",\n \"name\": \"8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:00.079720888Z\",\n \"endTime\": \"2026-10-20T05:17:05.395412324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\n correlation ID: 30ca1b52-8205-487a-9436-e632a6db61a6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x32189916ae0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-p4k6hc7mknvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\",\n \"name\": \"8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:00.079720888Z\",\n \"endTime\": \"2026-10-20T05:17:05.395412324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d0dc513-33ba-4d3d-8e9f-cd8439a0d699\n correlation ID: 30ca1b52-8205-487a-9436-e632a6db61a6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"39fc7ebfa26f2a6c8874741c3a29b7e196f7f2ea553eaf585f62c52974c5e68e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"ffaa5c86bbf96c4d215750cac34aa61535716a53507e4cbfbc11c0b6cf39ba38","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x75f66c1a3a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-4h26f2trg98s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\",\n \"name\": \"16db3e0e-b6cc-4636-b614-3ce9422b0cc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:35.421756418Z\",\n \"endTime\": \"2026-10-20T05:15:35.474351491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\n correlation ID: 68381922-028e-475b-8ed6-c1a70495ad96\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x75f66c1a3a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-4h26f2trg98s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\",\n \"name\": \"16db3e0e-b6cc-4636-b614-3ce9422b0cc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:35.421756418Z\",\n \"endTime\": \"2026-10-20T05:15:35.474351491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/16db3e0e-b6cc-4636-b614-3ce9422b0cc1\n correlation ID: 68381922-028e-475b-8ed6-c1a70495ad96\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3c07f513f8419736fa4ce53eb6b22e034ee6e6ee5f5b1d65d1c324a62ac12a32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"9e1ec5a9d28e9a47aa98f55666f7138cef8b72cca132d14571662467161fdedc","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-h9n588\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3267a40386c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-h9n588: failed waiting for cluster=\"cluster-zstream-4-20-h9n588\" in resourcegroup=\"rg-zstream-upgrade-4-20-fgbmk6fpcq6t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\",\n \"name\": \"74b7db42-8d43-4cbb-abcb-a49066e5f67c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:55.683730118Z\",\n \"endTime\": \"2026-10-20T06:04:00.540490052Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\n correlation ID: cbdbdf19-14bb-4c6b-b58c-17335053d977\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-h9n588\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x3267a40386c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-h9n588: failed waiting for cluster=\"cluster-zstream-4-20-h9n588\" in resourcegroup=\"rg-zstream-upgrade-4-20-fgbmk6fpcq6t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\",\n \"name\": \"74b7db42-8d43-4cbb-abcb-a49066e5f67c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:55.683730118Z\",\n \"endTime\": \"2026-10-20T06:04:00.540490052Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74b7db42-8d43-4cbb-abcb-a49066e5f67c\n correlation ID: cbdbdf19-14bb-4c6b-b58c-17335053d977\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45ba98c9265a842cf9a8176e0d8e67a79e9e727bbb57d5458019a36bab563aa2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"4ecbeef13a630701a21a74ce0404abcccf3cb96ab3a6bed45186772d392d1cbf","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x107be8a3e900\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pqxw4xjdlhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\",\n \"name\": \"49501419-c1fd-4026-844e-031c1a9806ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:13.525722235Z\",\n \"endTime\": \"2026-10-20T06:06:20.691561323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\n correlation ID: 00bab76d-860c-4424-93b4-54a3ab300a64\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x107be8a3e900\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pqxw4xjdlhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\",\n \"name\": \"49501419-c1fd-4026-844e-031c1a9806ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:13.525722235Z\",\n \"endTime\": \"2026-10-20T06:06:20.691561323Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49501419-c1fd-4026-844e-031c1a9806ef\n correlation ID: 00bab76d-860c-4424-93b4-54a3ab300a64\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4dfa5d583ceba033644eac8225e97ab20b2d9d80abc17a22bdeba8796cfa35b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"4c53253f9e54053e65f1d9f31d1f7c897f3ed64b8cd91d2d5a475876c4218f25","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-l95qzc\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x250cc094aa80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-l95qzc: failed waiting for cluster=\"cp-ystream-upgrade-4-23-l95qzc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-czxhkdjg5kq4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\",\n \"name\": \"0241f99d-3c51-4ff4-8382-31a87157a2aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:53.47570213Z\",\n \"endTime\": \"2026-10-20T06:06:00.655523868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\n correlation ID: f072941c-323b-4743-ab19-c84744480e30\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-l95qzc\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x250cc094aa80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-l95qzc: failed waiting for cluster=\"cp-ystream-upgrade-4-23-l95qzc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-czxhkdjg5kq4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\",\n \"name\": \"0241f99d-3c51-4ff4-8382-31a87157a2aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:53.47570213Z\",\n \"endTime\": \"2026-10-20T06:06:00.655523868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0241f99d-3c51-4ff4-8382-31a87157a2aa\n correlation ID: f072941c-323b-4743-ab19-c84744480e30\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4ff337e16be11a5f1b969af8235896a37eff6d20776b65f99d25b02cdde153cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"037ab5604e242e87117bb5a1f7a62b9304d6f5396451277b9fc3608601ed2669","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-jn77rzqmfkjz/cluster-candidate-5-1-j65tbc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x332b74a98940\u003e: \n failed to create HCP cluster cluster-candidate-5-1-j65tbc: failed waiting for cluster=\"cluster-candidate-5-1-j65tbc\" in resourcegroup=\"rg-candidate-5-1-jn77rzqmfkjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\",\n \"name\": \"be69220f-b32f-412c-9de4-3a1f140f5135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:23.131093305Z\",\n \"endTime\": \"2026-10-20T06:03:30.67205401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 9m40s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n correlation ID: 8aaf06ce-d046-4047-82a0-22fbcc3d8734\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-jn77rzqmfkjz/cluster-candidate-5-1-j65tbc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x332b74a98940\u003e: \n failed to create HCP cluster cluster-candidate-5-1-j65tbc: failed waiting for cluster=\"cluster-candidate-5-1-j65tbc\" in resourcegroup=\"rg-candidate-5-1-jn77rzqmfkjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\",\n \"name\": \"be69220f-b32f-412c-9de4-3a1f140f5135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:23.131093305Z\",\n \"endTime\": \"2026-10-20T06:03:30.67205401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 9m40s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n correlation ID: 8aaf06ce-d046-4047-82a0-22fbcc3d8734\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"60704c0e5cfebadfd1980b6ade8f7de8b5ad46e8b28f9ad60924807a07453d80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"3faaa107fa90727c96a34d72451750f7291409b311a0a4a67821b868f59cbab2","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x223a7b530de0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-nxdc7fm88rqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\",\n \"name\": \"d1cec42a-1f65-4e6d-b206-009980af3340\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:36.107714789Z\",\n \"endTime\": \"2026-10-20T05:15:45.396149502Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\n correlation ID: 1270fae9-4ae8-4595-afd1-32eaec691f46\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0x223a7b530de0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-nxdc7fm88rqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\",\n \"name\": \"d1cec42a-1f65-4e6d-b206-009980af3340\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:36.107714789Z\",\n \"endTime\": \"2026-10-20T05:15:45.396149502Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1cec42a-1f65-4e6d-b206-009980af3340\n correlation ID: 1270fae9-4ae8-4595-afd1-32eaec691f46\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"625da5810b81dfd6eefdbcc39d5c3030047039183d1cca8efdce2b1ede03951a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"3e55f008c63964d562d24005c7da675a204730845c588ff0e4ec44a7826514f3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-4ksqgjzzm96p/cluster-candidate-4-23-fwz5th should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x18cb9798c1e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-fwz5th: failed waiting for cluster=\"cluster-candidate-4-23-fwz5th\" in resourcegroup=\"rg-candidate-4-23-4ksqgjzzm96p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\",\n \"name\": \"6a6004b0-cbb7-4549-89e1-ac7033807043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:23.507594354Z\",\n \"endTime\": \"2026-10-20T06:03:30.699349661Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\n correlation ID: 7f202ac7-f74e-4cc2-9ec1-abe675d18359\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-4ksqgjzzm96p/cluster-candidate-4-23-fwz5th should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x18cb9798c1e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-fwz5th: failed waiting for cluster=\"cluster-candidate-4-23-fwz5th\" in resourcegroup=\"rg-candidate-4-23-4ksqgjzzm96p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\",\n \"name\": \"6a6004b0-cbb7-4549-89e1-ac7033807043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:23.507594354Z\",\n \"endTime\": \"2026-10-20T06:03:30.699349661Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6004b0-cbb7-4549-89e1-ac7033807043\n correlation ID: 7f202ac7-f74e-4cc2-9ec1-abe675d18359\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"66095641b133e6de8877dab06a2264d1f208e000ba71118d90f06b0406432703","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"b6c9b1a6dfebf2ea1ce6eab0f805c9997bb911c59710974cf5cf2a393c6b58df","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ab5c5948d60\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-t958jgbtjk8g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\",\n \"name\": \"f52797ae-d31d-47d6-bbc7-f205df02a324\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:27.324687165Z\",\n \"endTime\": \"2026-10-20T05:15:35.381667293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\n correlation ID: c3132aa9-5208-44f1-9e3d-37d8ebb54cd9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ab5c5948d60\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-t958jgbtjk8g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\",\n \"name\": \"f52797ae-d31d-47d6-bbc7-f205df02a324\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:27.324687165Z\",\n \"endTime\": \"2026-10-20T05:15:35.381667293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f52797ae-d31d-47d6-bbc7-f205df02a324\n correlation ID: c3132aa9-5208-44f1-9e3d-37d8ebb54cd9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6e33991d62709959269c182d93d3d6795c21197efbf5d46000e0d1b350bd3311","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"b821b9ea697aca4b87f94938289d6cb621ff01fdd2288c0cec68b2091b62a01d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f8f95e6a740\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4ghnv5cd7f58\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\",\n \"name\": \"830370c8-deae-4f2f-a0e6-e8cf99033394\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:15.093240534Z\",\n \"endTime\": \"2026-10-20T06:06:20.694573577Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\n correlation ID: 56a014be-31e7-444c-8c5c-d688c9ff202c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f8f95e6a740\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4ghnv5cd7f58\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\",\n \"name\": \"830370c8-deae-4f2f-a0e6-e8cf99033394\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:15.093240534Z\",\n \"endTime\": \"2026-10-20T06:06:20.694573577Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/830370c8-deae-4f2f-a0e6-e8cf99033394\n correlation ID: 56a014be-31e7-444c-8c5c-d688c9ff202c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6e39a2744bc82d052649e6b6d0d57c5e45ad44d64107055bf7bf3efdd5264ffe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"ed4d2ea5ea2e59de470ab104b735900451489e3a699014d5642dd9a2cd16beef","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hm52w6 with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xed224e60cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-hm52w6: failed waiting for cluster=\"np-version-upgrade-cluster-hm52w6\" in resourcegroup=\"rg-np-version-upgrade-hm52w6-wx74vzbhpsbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\",\n \"name\": \"1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:18.096329858Z\",\n \"endTime\": \"2026-10-20T06:54:20.326558376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, ingress-operator, redhat-marketplace-catalog, dns-operator, hosted-cluster-config-operator, community-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, konnectivity-agent, catalog-operator, olm-operator, redhat-operators-catalog, kube-controller-manager, kube-scheduler, cluster-version-operator, cluster-node-tuning-operator, cluster-storage-operator, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\n correlation ID: 629b9eb2-a5ed-4ab1-8b09-2a2926dfaf45\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hm52w6 with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xed224e60cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-hm52w6: failed waiting for cluster=\"np-version-upgrade-cluster-hm52w6\" in resourcegroup=\"rg-np-version-upgrade-hm52w6-wx74vzbhpsbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\",\n \"name\": \"1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:18.096329858Z\",\n \"endTime\": \"2026-10-20T06:54:20.326558376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, ingress-operator, redhat-marketplace-catalog, dns-operator, hosted-cluster-config-operator, community-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, konnectivity-agent, catalog-operator, olm-operator, redhat-operators-catalog, kube-controller-manager, kube-scheduler, cluster-version-operator, cluster-node-tuning-operator, cluster-storage-operator, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1eeebc3a-c4ef-4aef-9df3-1a9f3554a850\n correlation ID: 629b9eb2-a5ed-4ab1-8b09-2a2926dfaf45\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6e9cf772c54e41f29be59c88f4d5ed3ed702937445893c0116f84c7067efa187","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"6f9537cede3126ce1bf541078c4af085af4c75317ec382bbf40d8c920322279b","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e9e304bc800\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hrm4cs7zffg2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n \"name\": \"a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:15.624065092Z\",\n \"endTime\": \"2026-10-20T05:15:15.524291644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n correlation ID: 8327c480-3421-4e24-a66c-3cea0477dfa6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e9e304bc800\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hrm4cs7zffg2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n \"name\": \"a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:15.624065092Z\",\n \"endTime\": \"2026-10-20T05:15:15.524291644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n correlation ID: 8327c480-3421-4e24-a66c-3cea0477dfa6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8bfd122bfb355a24d92287df3269e0a7f0e6a9d4d45c80a03f8103457622a6b5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7480db79c3e887a1355f096e64e33280237bde28f8a142a93764f3bc40cc0607","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"136f3e529c41bdef3473933d2081faf92c073f86ca46c938c2727073e0dee688","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-m5qdzq\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x15fef97f26c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-m5qdzq: failed waiting for cluster=\"cp-ystream-upgrade-4-22-m5qdzq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-qpfdsx5knlp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\",\n \"name\": \"1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:50.514339704Z\",\n \"endTime\": \"2026-10-20T06:05:50.51641013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-policy-controller, catalog-operator, olm-operator, ingress-operator, konnectivity-agent, cluster-autoscaler, machine-approver, control-plane-pki-operator, kube-scheduler, cluster-version-operator, azure-cloud-controller-manager, olm-collect-profiles, capi-provider, cluster-network-operator, openshift-apiserver, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, certified-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\n correlation ID: b0ed391a-525b-46bf-89dd-2398b6d1d42a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-m5qdzq\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x15fef97f26c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-m5qdzq: failed waiting for cluster=\"cp-ystream-upgrade-4-22-m5qdzq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-qpfdsx5knlp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\",\n \"name\": \"1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:50.514339704Z\",\n \"endTime\": \"2026-10-20T06:05:50.51641013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-policy-controller, catalog-operator, olm-operator, ingress-operator, konnectivity-agent, cluster-autoscaler, machine-approver, control-plane-pki-operator, kube-scheduler, cluster-version-operator, azure-cloud-controller-manager, olm-collect-profiles, capi-provider, cluster-network-operator, openshift-apiserver, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, certified-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ff8ed3a-d241-4f3b-855b-4d5b91e2266e\n correlation ID: b0ed391a-525b-46bf-89dd-2398b6d1d42a\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7a3bb2deb63f67e81e7c6445d077cb0592f0cc4618ab3ab5b025e90d6b91fc9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"5bed8bd505b275faee3a7c65055544105675a2bbfc9f487e68e44672dfb62806","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x87fabdfe6a0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nb9lb62595s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n \"name\": \"879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:20:18.481707716Z\",\n \"endTime\": \"2026-10-20T06:49:20.225008169Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n correlation ID: cf264c90-6764-45bb-a904-1799b612e538\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x87fabdfe6a0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nb9lb62595s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n \"name\": \"879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:20:18.481707716Z\",\n \"endTime\": \"2026-10-20T06:49:20.225008169Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n correlation ID: cf264c90-6764-45bb-a904-1799b612e538\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a841661ff321264fc6d63199430ded1478aee9bbad0f27351d0c98a58e041b68","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7baebceea490b36d55a4a8efe6dd8c99a8f838f612f2cb552f426749bf1e9e3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"2eb8f17d8e8fdf3ef65026f596d61ddd2edf43214a5d339daac7727d062002fe","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x21bf8fa52440\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-45l6zfp27xp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\",\n \"name\": \"76bd04b2-6b59-43d9-8550-04729b1e23a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:15.893774607Z\",\n \"endTime\": \"2026-10-20T05:15:15.422936385Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\n correlation ID: 957f5a51-47b9-4bd5-baae-b42b2556edad\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x21bf8fa52440\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-45l6zfp27xp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\",\n \"name\": \"76bd04b2-6b59-43d9-8550-04729b1e23a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:15.893774607Z\",\n \"endTime\": \"2026-10-20T05:15:15.422936385Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/76bd04b2-6b59-43d9-8550-04729b1e23a8\n correlation ID: 957f5a51-47b9-4bd5-baae-b42b2556edad\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7be88fb93b0359f8b2eb054022c9e5c10ff9f8e709f7f6db292873a428fec42b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"f63a6ebbc5501e9879294a8f133573498b6e673dcf7325d2092848e0d64f6ab8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-20-6rcjmk8jgsnk/cluster-candidate-4-20-nk5kcm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb675fee720\u003e: \n failed to create HCP cluster cluster-candidate-4-20-nk5kcm: failed waiting for cluster=\"cluster-candidate-4-20-nk5kcm\" in resourcegroup=\"rg-candidate-4-20-6rcjmk8jgsnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\",\n \"name\": \"f3aff00b-ac70-4419-97bf-0db3c5913620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:25:32.583784401Z\",\n \"endTime\": \"2026-10-20T05:54:41.597043343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, cluster-storage-operator, packageserver, hosted-cluster-config-operator, dns-operator, certified-operators-catalog, redhat-marketplace-catalog, kube-controller-manager, openshift-route-controller-manager, cluster-network-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\n correlation ID: 01a97c61-6d3c-4aab-8c8e-ff7b2d9d9897\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-20-6rcjmk8jgsnk/cluster-candidate-4-20-nk5kcm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb675fee720\u003e: \n failed to create HCP cluster cluster-candidate-4-20-nk5kcm: failed waiting for cluster=\"cluster-candidate-4-20-nk5kcm\" in resourcegroup=\"rg-candidate-4-20-6rcjmk8jgsnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\",\n \"name\": \"f3aff00b-ac70-4419-97bf-0db3c5913620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:25:32.583784401Z\",\n \"endTime\": \"2026-10-20T05:54:41.597043343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, cluster-storage-operator, packageserver, hosted-cluster-config-operator, dns-operator, certified-operators-catalog, redhat-marketplace-catalog, kube-controller-manager, openshift-route-controller-manager, cluster-network-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f3aff00b-ac70-4419-97bf-0db3c5913620\n correlation ID: 01a97c61-6d3c-4aab-8c8e-ff7b2d9d9897\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7e8dceb6236a3e7ee33501578d4bec31aa676b52d57de55a5a447292807fb713","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d8b29925f38a23319a5e29ac21fd486c6701e6e44b71c79797bb3944c415017e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xb18f90464a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h2n57626mmtj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\",\n \"name\": \"0293c355-6abe-4d44-a125-21fadb017776\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:31.672390491Z\",\n \"endTime\": \"2026-10-20T05:15:35.46985478Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\n correlation ID: 587e1c35-20e5-4d6f-b21e-d73f4e20faf9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xb18f90464a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h2n57626mmtj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\",\n \"name\": \"0293c355-6abe-4d44-a125-21fadb017776\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:31.672390491Z\",\n \"endTime\": \"2026-10-20T05:15:35.46985478Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0293c355-6abe-4d44-a125-21fadb017776\n correlation ID: 587e1c35-20e5-4d6f-b21e-d73f4e20faf9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"810ebda7e273946904138871ae64d464ea7f792a03a6f06f6c6132365c933053","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"f96c57f99ac73af4b4c3514a5ad2d2bc098868a52ebe93532ace21aa2781fffd","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x28688821ac40\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-69bxttgtnjdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\",\n \"name\": \"81fcc58f-c93e-4c14-8462-568e76b1f007\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:44.602488636Z\",\n \"endTime\": \"2026-10-20T06:58:50.194412463Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.w4p3g9f2y8m5o4t.2kot.eastus2euap.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n correlation ID: 57aab543-83b3-413d-b1ad-731d8e533b5f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x28688821ac40\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-69bxttgtnjdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\",\n \"name\": \"81fcc58f-c93e-4c14-8462-568e76b1f007\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:44.602488636Z\",\n \"endTime\": \"2026-10-20T06:58:50.194412463Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.w4p3g9f2y8m5o4t.2kot.eastus2euap.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n correlation ID: 57aab543-83b3-413d-b1ad-731d8e533b5f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"600ad974be9c7777150a3432faf4397d3733a0f1f567bf0b6ee9a4202295378e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"854cfbd4e8f8046f5be59c24f53d09ca77ac91cf276a6b24e2f629162833232d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"49bc5863be8707ea32e900c899a764b317035418945da414b2fd1bb27cbd6874","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x3609409b2da0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-k6pr5pngbf2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\",\n \"name\": \"4b8de532-806b-4184-a7a5-d708e688d877\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:17.851260382Z\",\n \"endTime\": \"2026-10-20T05:15:25.510242147Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\n correlation ID: 3e8d5d1e-8572-4078-b33e-e166b2056a38\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x3609409b2da0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-k6pr5pngbf2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\",\n \"name\": \"4b8de532-806b-4184-a7a5-d708e688d877\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:17.851260382Z\",\n \"endTime\": \"2026-10-20T05:15:25.510242147Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b8de532-806b-4184-a7a5-d708e688d877\n correlation ID: 3e8d5d1e-8572-4078-b33e-e166b2056a38\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"867059ad4ec220f20e1ba7e4a67b93bf0b4f0c0455432d44ddfcdf597abfe18d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"5a163e5e0a83014f555d69d77aa75af4bf1868419228ae0ae4dfd4a51baa6b7c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-fjbzvxbbtgch/cluster-candidate-4-21-qjz2fr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x280193782d80\u003e: \n failed to create HCP cluster cluster-candidate-4-21-qjz2fr: failed waiting for cluster=\"cluster-candidate-4-21-qjz2fr\" in resourcegroup=\"rg-candidate-4-21-fjbzvxbbtgch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\",\n \"name\": \"1226a502-bdfe-4d84-9939-b5ba30a000ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:26:47.973858758Z\",\n \"endTime\": \"2026-10-20T05:55:51.546352247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, olm-collect-profiles, certified-operators-catalog, openshift-controller-manager, hosted-cluster-config-operator, konnectivity-agent, catalog-operator, cluster-image-registry-operator, capi-provider, openshift-apiserver, azure-cloud-controller-manager, redhat-operators-catalog, machine-approver, control-plane-pki-operator, kube-scheduler, packageserver, redhat-marketplace-catalog, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, community-operators-catalog, ingress-operator, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\n correlation ID: 89c9a7f5-600b-433c-814f-c8cadcd9f923\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-fjbzvxbbtgch/cluster-candidate-4-21-qjz2fr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x280193782d80\u003e: \n failed to create HCP cluster cluster-candidate-4-21-qjz2fr: failed waiting for cluster=\"cluster-candidate-4-21-qjz2fr\" in resourcegroup=\"rg-candidate-4-21-fjbzvxbbtgch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\",\n \"name\": \"1226a502-bdfe-4d84-9939-b5ba30a000ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:26:47.973858758Z\",\n \"endTime\": \"2026-10-20T05:55:51.546352247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, olm-collect-profiles, certified-operators-catalog, openshift-controller-manager, hosted-cluster-config-operator, konnectivity-agent, catalog-operator, cluster-image-registry-operator, capi-provider, openshift-apiserver, azure-cloud-controller-manager, redhat-operators-catalog, machine-approver, control-plane-pki-operator, kube-scheduler, packageserver, redhat-marketplace-catalog, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, community-operators-catalog, ingress-operator, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1226a502-bdfe-4d84-9939-b5ba30a000ab\n correlation ID: 89c9a7f5-600b-433c-814f-c8cadcd9f923\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86d94196c3c501fad47bd7b85271c9d658b5b9fa6491f79b14c259c353a1480d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d706d1ccca667e86b44aee723bd6c4d7339497578c8a703fc65f912c293de8ea","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-s5dcsw\" with version 4.22.18 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x15e376eb4ba0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-s5dcsw: failed waiting for cluster=\"cluster-zstream-4-22-s5dcsw\" in resourcegroup=\"rg-zstream-upgrade-4-22-t6t94hh7j44m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\",\n \"name\": \"08e66858-550b-405f-b39d-e72d4d3457b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:35:01.18923569Z\",\n \"endTime\": \"2026-10-20T06:04:10.578239766Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, kube-storage-version-migrator, azure-cloud-controller-manager, packageserver, community-operators-catalog, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, openshift-controller-manager, olm-collect-profiles, kube-controller-manager, kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-operators-catalog, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n correlation ID: feffceb9-72b0-45a3-8f5c-db4fbe3e6004\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-s5dcsw\" with version 4.22.18 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x15e376eb4ba0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-s5dcsw: failed waiting for cluster=\"cluster-zstream-4-22-s5dcsw\" in resourcegroup=\"rg-zstream-upgrade-4-22-t6t94hh7j44m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\",\n \"name\": \"08e66858-550b-405f-b39d-e72d4d3457b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:35:01.18923569Z\",\n \"endTime\": \"2026-10-20T06:04:10.578239766Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, kube-storage-version-migrator, azure-cloud-controller-manager, packageserver, community-operators-catalog, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, openshift-controller-manager, olm-collect-profiles, kube-controller-manager, kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-operators-catalog, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n correlation ID: feffceb9-72b0-45a3-8f5c-db4fbe3e6004\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"871573048dd24efa8881069655e740b4d1c0d382d4b32b4f414af3de7bc11b2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"a5bc08bcb1e4e6dcd7e514e3424f3eb0c78f0cbb642a2952ccefe4b2b4c6b3e2","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x2597f46154e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cxb2nzvthpwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\",\n \"name\": \"023dabe4-0169-483a-88ae-578717bb647a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:49:31.280853447Z\",\n \"endTime\": \"2026-10-20T05:18:35.3945667Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\n correlation ID: 1a834fe8-01d3-4884-9266-9d76c40880d5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x2597f46154e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cxb2nzvthpwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\",\n \"name\": \"023dabe4-0169-483a-88ae-578717bb647a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:49:31.280853447Z\",\n \"endTime\": \"2026-10-20T05:18:35.3945667Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/023dabe4-0169-483a-88ae-578717bb647a\n correlation ID: 1a834fe8-01d3-4884-9266-9d76c40880d5\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"912dc081d5aba61fe76513fc626e09c01f6b17028fe2cf320c2cdff1acb0152e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"a464e881b3b5384903bbf8b5cba5cbec43270a9c169c5298ecd2ca63b6863f35","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b92f7aa27e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-rc927glkdq2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\",\n \"name\": \"a4238e21-3b2a-42f9-bff0-aa4a966521a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:16.780348906Z\",\n \"endTime\": \"2026-10-20T06:58:20.197046353Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-marketplace-catalog, machine-approver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, kube-controller-manager, olm-operator, openshift-controller-manager, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\n correlation ID: a20a43f8-bc44-4b37-9a04-ac469e2a7724\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b92f7aa27e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-rc927glkdq2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\",\n \"name\": \"a4238e21-3b2a-42f9-bff0-aa4a966521a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:16.780348906Z\",\n \"endTime\": \"2026-10-20T06:58:20.197046353Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-marketplace-catalog, machine-approver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, kube-controller-manager, olm-operator, openshift-controller-manager, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a4238e21-3b2a-42f9-bff0-aa4a966521a1\n correlation ID: a20a43f8-bc44-4b37-9a04-ac469e2a7724\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"938768ffc7a9202f6ea2a020681436deedb71c75d8d1174e02b7dbfca31abc1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"4a8ff495293973ab2315985d325a1ead08660f0d23a49dee4c3f8735bfcc49eb","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x9ffbe0a6720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-c2tg56ndlf7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n \"name\": \"862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.770566501Z\",\n \"endTime\": \"2026-10-20T05:15:25.516223632Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-storage-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, catalog-operator, cluster-version-operator, ingress-operator, olm-operator, openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n correlation ID: d433d7b9-c808-4842-80b1-be90eb0e3f4a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x9ffbe0a6720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-c2tg56ndlf7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n \"name\": \"862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.770566501Z\",\n \"endTime\": \"2026-10-20T05:15:25.516223632Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-storage-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, catalog-operator, cluster-version-operator, ingress-operator, olm-operator, openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n correlation ID: d433d7b9-c808-4842-80b1-be90eb0e3f4a\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"60b366d51c10b5b9b70cb40285af4daf6406290c1d165863a793e90acfa07ec5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9bdfa7e47022aed847d5f3ca123b702c77edca0dce3f238b5945eb0ed202a288","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"5ce5e5f754031dd5e2b520b97795f008263e7e8ee14634915d4fd663395567e2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dc51d83d2c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dsxq6jxzmbtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\",\n \"name\": \"30bbed47-9a91-40fe-b99b-caa97b286f39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:24.928973066Z\",\n \"endTime\": \"2026-10-20T05:15:25.520014214Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n correlation ID: d0e5fc28-aced-4621-9518-831e81cdc82c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dc51d83d2c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dsxq6jxzmbtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\",\n \"name\": \"30bbed47-9a91-40fe-b99b-caa97b286f39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:24.928973066Z\",\n \"endTime\": \"2026-10-20T05:15:25.520014214Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n correlation ID: d0e5fc28-aced-4621-9518-831e81cdc82c\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a841661ff321264fc6d63199430ded1478aee9bbad0f27351d0c98a58e041b68","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c8b32596bca30d693eb93f0d5dd3048a0e6b2171a1d5733241f144b77bedc5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"39cb4b853fb39042d89ad859200027cc772c814433d47a8e1f5499bd7bdff16c","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d91e77ae5e0\u003e: \n failed to create HCP cluster np-downgrade-pqjrkt: failed waiting for cluster=\"np-downgrade-pqjrkt\" in resourcegroup=\"rg-np-downgrade-pqjrkt-mjxxp87f7rvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\",\n \"name\": \"33e1ecfd-a213-4c18-bb3d-e2e750131edc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:28:43.42016767Z\",\n \"endTime\": \"2026-10-20T06:57:50.254500429Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-operators-catalog, openshift-controller-manager, dns-operator, ingress-operator, catalog-operator, cluster-node-tuning-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\n correlation ID: e485c2ec-c71d-4dd8-bff1-9c5e356c91a6\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d91e77ae5e0\u003e: \n failed to create HCP cluster np-downgrade-pqjrkt: failed waiting for cluster=\"np-downgrade-pqjrkt\" in resourcegroup=\"rg-np-downgrade-pqjrkt-mjxxp87f7rvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\",\n \"name\": \"33e1ecfd-a213-4c18-bb3d-e2e750131edc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:28:43.42016767Z\",\n \"endTime\": \"2026-10-20T06:57:50.254500429Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-operators-catalog, openshift-controller-manager, dns-operator, ingress-operator, catalog-operator, cluster-node-tuning-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, cluster-image-registry-operator, konnectivity-agent, olm-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33e1ecfd-a213-4c18-bb3d-e2e750131edc\n correlation ID: e485c2ec-c71d-4dd8-bff1-9c5e356c91a6\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d742ef35abd1cd1925d9860cb84566f3a1f0337e102ed31bd5baede03f62834","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"3b304d4ecdb8dc3eda09b0dd43c3474f5ca2580737f170b0f34deb2e5dd22d25","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-xc4xw9\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e9c1dd82580\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xc4xw9: failed waiting for cluster=\"cluster-zstream-5-0-xc4xw9\" in resourcegroup=\"rg-zstream-upgrade-5-0-pmckdddm6qmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\",\n \"name\": \"e7a4e06c-22d7-457b-b388-43a17797d319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:00.234554188Z\",\n \"endTime\": \"2026-10-20T06:05:00.517636269Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\n correlation ID: c2c66cf5-1898-42eb-bd67-31f44f50f0f3\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-xc4xw9\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e9c1dd82580\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xc4xw9: failed waiting for cluster=\"cluster-zstream-5-0-xc4xw9\" in resourcegroup=\"rg-zstream-upgrade-5-0-pmckdddm6qmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\",\n \"name\": \"e7a4e06c-22d7-457b-b388-43a17797d319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:00.234554188Z\",\n \"endTime\": \"2026-10-20T06:05:00.517636269Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7a4e06c-22d7-457b-b388-43a17797d319\n correlation ID: c2c66cf5-1898-42eb-bd67-31f44f50f0f3\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a1677d492b0d8d54e87cb1d00af6bb16e90249d93b396ef9ea43683804c2591e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d3b3192ea30a5bd5e470657add4a5cc9c892b2e2b76b49698e1203d4b6060403","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e05109c4140\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-7cbm7pwl25bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n \"name\": \"c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:17:40.509470506Z\",\n \"endTime\": \"2026-10-20T06:46:40.233866938Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n correlation ID: 3a5074fe-d20a-46db-b752-31f0ad8d374b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e05109c4140\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-7cbm7pwl25bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n \"name\": \"c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:17:40.509470506Z\",\n \"endTime\": \"2026-10-20T06:46:40.233866938Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n correlation ID: 3a5074fe-d20a-46db-b752-31f0ad8d374b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ecbe2fdf10383ad3e6b4b9603afc5bfad8be6ed12177f6a3782005a280d5e7a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ignition-server-proxy, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a2b4398b851278157ac16b1ec57cf79ef5f3106de3a4028c660034cb14c1bba5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"30230279c2e3510d994380632c7a574f5e6af12cc1a0c4498c3b30dd95687f47","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-9kxpfb\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c2213338760\u003e: \n failed to create HCP cluster np-noedge-9kxpfb: failed waiting for cluster=\"np-noedge-9kxpfb\" in resourcegroup=\"rg-np-noedge-9kxpfb-sml22h7w7mpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\",\n \"name\": \"a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:35.454086291Z\",\n \"endTime\": \"2026-10-20T06:54:40.265150731Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-image-registry-operator, redhat-operators-catalog, openshift-route-controller-manager, ingress-operator, community-operators-catalog, capi-provider, kube-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, catalog-operator, machine-approver, cluster-version-operator, cluster-policy-controller, olm-operator, openshift-apiserver, hosted-cluster-config-operator, olm-collect-profiles, control-plane-pki-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, konnectivity-agent, packageserver, certified-operators-catalog, cluster-network-operator, redhat-marketplace-catalog, openshift-controller-manager, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\n correlation ID: 7ad8badf-d599-4a68-a634-9e4a02f000bc\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-9kxpfb\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c2213338760\u003e: \n failed to create HCP cluster np-noedge-9kxpfb: failed waiting for cluster=\"np-noedge-9kxpfb\" in resourcegroup=\"rg-np-noedge-9kxpfb-sml22h7w7mpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\",\n \"name\": \"a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:25:35.454086291Z\",\n \"endTime\": \"2026-10-20T06:54:40.265150731Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-image-registry-operator, redhat-operators-catalog, openshift-route-controller-manager, ingress-operator, community-operators-catalog, capi-provider, kube-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, catalog-operator, machine-approver, cluster-version-operator, cluster-policy-controller, olm-operator, openshift-apiserver, hosted-cluster-config-operator, olm-collect-profiles, control-plane-pki-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, konnectivity-agent, packageserver, certified-operators-catalog, cluster-network-operator, redhat-marketplace-catalog, openshift-controller-manager, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a5f83473-bbd4-4ad3-bec9-bd943e18d7a5\n correlation ID: 7ad8badf-d599-4a68-a634-9e4a02f000bc\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a70e2d8140812016e752d225266d36d2937281cebbc0a361d6bf9280026151a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"859ea0d5c5bc8abfe7fd4ff75ecbc3fb7c86fc5e4d3a9f2febaf8197f05ec208","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x90ed851c7c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pktl7x9f5vkg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\",\n \"name\": \"a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:04.743864123Z\",\n \"endTime\": \"2026-10-20T05:17:05.401635913Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\n correlation ID: 02b60666-e337-48ba-b354-abcca45fd7b3\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x90ed851c7c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pktl7x9f5vkg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\",\n \"name\": \"a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:04.743864123Z\",\n \"endTime\": \"2026-10-20T05:17:05.401635913Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a09ea7c2-a4ae-4b3a-928f-fdd5cb63fa7e\n correlation ID: 02b60666-e337-48ba-b354-abcca45fd7b3\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ac76b419a2db9a59cb4c86c1e001e9008c9e74a0f6b22b622b6ed51f6e108c2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"5953dd256eb587494cc84cbe3d24465e66cade2a891fa9dea6669db6066c85b2","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2011c863ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-5m565ljzrpfk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\",\n \"name\": \"dbfe9b86-5500-4c33-9dca-ce294ebba251\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.618408533Z\",\n \"endTime\": \"2026-10-20T05:15:25.394652069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\n correlation ID: 3e755b64-47a8-4661-a7bb-bcc8f775f088\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2011c863ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-5m565ljzrpfk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\",\n \"name\": \"dbfe9b86-5500-4c33-9dca-ce294ebba251\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.618408533Z\",\n \"endTime\": \"2026-10-20T05:15:25.394652069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dbfe9b86-5500-4c33-9dca-ce294ebba251\n correlation ID: 3e755b64-47a8-4661-a7bb-bcc8f775f088\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b01124dfac813b1f20646afbc719cadfc45ad5d486cf3be983b9fedd4dd72d73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d53c47891282a5e167e06922b6c9850dc9d2bd94d5c1cd79f7cd49d1e12a8731","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x15c6e3a04a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-dxxspkcjk26g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\",\n \"name\": \"ca92c359-4f98-4827-b7f8-7ac271593322\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:24.892078047Z\",\n \"endTime\": \"2026-10-20T05:15:25.52669287Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\n correlation ID: 76cf023f-9041-4a5e-ba84-15f113c26157\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x15c6e3a04a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-dxxspkcjk26g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\",\n \"name\": \"ca92c359-4f98-4827-b7f8-7ac271593322\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:24.892078047Z\",\n \"endTime\": \"2026-10-20T05:15:25.52669287Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ca92c359-4f98-4827-b7f8-7ac271593322\n correlation ID: 76cf023f-9041-4a5e-ba84-15f113c26157\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b19d45a2942e6313d168c38aaf27f1f0cb9859e39b0de8d44b50fe4e71fe2b56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"b4b2911a14637b8143295f977cecd07da7b4485d97e16f41e9d56ad85e51f1ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"ac10ce2cee387f866229f3daf6fd76c6a13c92110bcadbc1ee7e47af615dc842","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ab9a24dc520\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gvhnhk5p9nvn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\",\n \"name\": \"b2fb4e18-00a4-4ae1-b373-5a45b63cc180\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.380774501Z\",\n \"endTime\": \"2026-10-20T05:15:25.399738527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\n correlation ID: 979533bf-931b-4b92-a315-5fa98c872193\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ab9a24dc520\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gvhnhk5p9nvn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\",\n \"name\": \"b2fb4e18-00a4-4ae1-b373-5a45b63cc180\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.380774501Z\",\n \"endTime\": \"2026-10-20T05:15:25.399738527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2fb4e18-00a4-4ae1-b373-5a45b63cc180\n correlation ID: 979533bf-931b-4b92-a315-5fa98c872193\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b8ed7c9fee036106dc76b66c5044170b7f00a9e2e0e16d212481052dbaebd188","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"b63af5629d6d99f87cae38122036785f946e70231cea7b91bbaea3046bb3156c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-sb5hdj\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x14e198164b40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-sb5hdj: failed waiting for cluster=\"cp-ystream-upgrade-5-1-sb5hdj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-p2s6v867jfzc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\",\n \"name\": \"231d826e-501b-479d-ad2c-3b14086d7685\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:49.284137334Z\",\n \"endTime\": \"2026-10-20T06:05:50.706441863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\n correlation ID: f4eb2870-85c2-458c-89db-87eaddaa1e62\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-sb5hdj\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x14e198164b40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-sb5hdj: failed waiting for cluster=\"cp-ystream-upgrade-5-1-sb5hdj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-p2s6v867jfzc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\",\n \"name\": \"231d826e-501b-479d-ad2c-3b14086d7685\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:49.284137334Z\",\n \"endTime\": \"2026-10-20T06:05:50.706441863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/231d826e-501b-479d-ad2c-3b14086d7685\n correlation ID: f4eb2870-85c2-458c-89db-87eaddaa1e62\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bbec8b89fdcc4de4166dee9968daaab3f3bcbab27b0e7ab2f8b4d0b7c525c796","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"0719c23731937943a86a2b3aa9a99194c166c5e5b9d50721b270d5435f98088d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x7c2c56e0ce0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-68cgxqrnf77h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\",\n \"name\": \"7543a13f-d4ca-45e0-8cd3-0ce47e921945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.001819115Z\",\n \"endTime\": \"2026-10-20T05:15:25.401147446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, control-plane-pki-operator, openshift-controller-manager, cluster-network-operator, cluster-storage-operator, ingress-operator, certified-operators-catalog, csi-snapshot-controller-operator, olm-operator, machine-approver, kube-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver, redhat-operators-catalog, cluster-policy-controller, cluster-version-operator, dns-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, catalog-operator, cluster-autoscaler, capi-provider, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\n correlation ID: ef9b51b6-03a3-4916-8005-c7bd18d6341f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x7c2c56e0ce0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-68cgxqrnf77h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\",\n \"name\": \"7543a13f-d4ca-45e0-8cd3-0ce47e921945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:46:25.001819115Z\",\n \"endTime\": \"2026-10-20T05:15:25.401147446Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, control-plane-pki-operator, openshift-controller-manager, cluster-network-operator, cluster-storage-operator, ingress-operator, certified-operators-catalog, csi-snapshot-controller-operator, olm-operator, machine-approver, kube-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver, redhat-operators-catalog, cluster-policy-controller, cluster-version-operator, dns-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, catalog-operator, cluster-autoscaler, capi-provider, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7543a13f-d4ca-45e0-8cd3-0ce47e921945\n correlation ID: ef9b51b6-03a3-4916-8005-c7bd18d6341f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bcbc7689bec8f0dfbedc0bd668b746abb59f95db4d471cc3df2b535e8cac7782","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d43e5e329e630584b37978546c0bf82691057e6c069848ea21832f0299660b4e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bda3528c680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxsg6grsgkll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n \"name\": \"2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:23:14.734977136Z\",\n \"endTime\": \"2026-10-20T05:52:21.558247967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n correlation ID: 2af21d6c-711a-4dc5-b3d3-22e3f2257e3e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bda3528c680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxsg6grsgkll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n \"name\": \"2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:23:14.734977136Z\",\n \"endTime\": \"2026-10-20T05:52:21.558247967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n correlation ID: 2af21d6c-711a-4dc5-b3d3-22e3f2257e3e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4fad994d0f98f503f348d9fe7df4759af2e675b2b5dfd38e9ccf9fd05ba39ea","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c828add128e99f73d7a3c3dcb8d919d2573e73860c17f33da78fcad57199d995","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"8cb5b55f31703889227570c650157a29009d93bae37355a94b6d3d03cda9c20b","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xae91b2b14a0\u003e: \n failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-5zjb4vwqmp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\",\n \"name\": \"74351f12-7936-4804-bc6e-44e43566ba15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:02.298920957Z\",\n \"endTime\": \"2026-10-20T06:06:10.689160757Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 13m18s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n correlation ID: 139f95f2-0f80-424c-9b9b-7d135360bb2a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xae91b2b14a0\u003e: \n failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-5zjb4vwqmp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\",\n \"name\": \"74351f12-7936-4804-bc6e-44e43566ba15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:02.298920957Z\",\n \"endTime\": \"2026-10-20T06:06:10.689160757Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 13m18s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n correlation ID: 139f95f2-0f80-424c-9b9b-7d135360bb2a\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ca3ba00ad0949e7eefffc70c855a4dfffab6a99aae67e4f022a289b490895b63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d756b1de5d0f247e4ca4acfc5768973e04b86c119eb06d43ef0dd144ba7446e0","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x3a0f21760840\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-m69f8px6srdw\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x3a0f2173b0c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x3a0f21760840\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-m69f8px6srdw\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x3a0f2173b0c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded"}},{"row_id":"ce86812fed034f4506e29e9d21d3e6af927a6bd4ab39c5d1f9e2c20aa7795a88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"3a4bdebb5a54840d29dd533497f38dba7690806bfcea4d2557d63b7ca4439497","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-7nhnvx at version 4.22.19\nUnexpected error:\n \u003c*fmt.wrapError | 0x21dd24a87460\u003e: \n failed to create HCP cluster np-skip-minor-7nhnvx: failed waiting for cluster=\"np-skip-minor-7nhnvx\" in resourcegroup=\"rg-np-skip-minor-7nhnvx-rt7mt2mm9mfc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\",\n \"name\": \"b182150e-6d0b-415e-b794-66f34e9c7538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:26:38.471608255Z\",\n \"endTime\": \"2026-10-20T06:55:40.175823021Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-collect-profiles, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, ingress-operator, cluster-autoscaler, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, konnectivity-agent, kube-storage-version-migrator, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\n correlation ID: 984419ff-41ae-498e-adef-4bd686e5fbe0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-7nhnvx at version 4.22.19\nUnexpected error:\n \u003c*fmt.wrapError | 0x21dd24a87460\u003e: \n failed to create HCP cluster np-skip-minor-7nhnvx: failed waiting for cluster=\"np-skip-minor-7nhnvx\" in resourcegroup=\"rg-np-skip-minor-7nhnvx-rt7mt2mm9mfc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\",\n \"name\": \"b182150e-6d0b-415e-b794-66f34e9c7538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:26:38.471608255Z\",\n \"endTime\": \"2026-10-20T06:55:40.175823021Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-collect-profiles, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, ingress-operator, cluster-autoscaler, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, konnectivity-agent, kube-storage-version-migrator, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b182150e-6d0b-415e-b794-66f34e9c7538\n correlation ID: 984419ff-41ae-498e-adef-4bd686e5fbe0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d279c743b0c258279f8d6e67359669d4e10648fa89be5c8717dda0623e74b419","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"9e3c71a0b8f48edf31f81234c87ded5a299025a3d5e7bd40495202e2fd2aa54e","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f03b61ccac0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xkdl5ldkwkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\",\n \"name\": \"60150ecc-c0c8-4ddc-abff-d4417c838152\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:17:57.832413824Z\",\n \"endTime\": \"2026-10-20T06:47:00.220229717Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n correlation ID: 9cce0d52-7c81-4f97-a269-e15c1e2fd419\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f03b61ccac0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xkdl5ldkwkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\",\n \"name\": \"60150ecc-c0c8-4ddc-abff-d4417c838152\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:17:57.832413824Z\",\n \"endTime\": \"2026-10-20T06:47:00.220229717Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n correlation ID: 9cce0d52-7c81-4f97-a269-e15c1e2fd419\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4fad994d0f98f503f348d9fe7df4759af2e675b2b5dfd38e9ccf9fd05ba39ea","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d9f6d6d92de2590a146dd2aa063a1ec90a70432597b353f3aa4e8d3cdc9c1cbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d3de213c99043332e16e77edea115f0ad9cddb7a0829c21a9a186e31e21883c7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x218ff96e360\u003e: \n failed to create HCP cluster np-dg-minor-grlgd9: failed waiting for cluster=\"np-dg-minor-grlgd9\" in resourcegroup=\"rg-np-dg-minor-grlgd9-gr8jq47glbkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\",\n \"name\": \"648800c4-40f0-4f5e-9d03-bc80720958e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:28.955511783Z\",\n \"endTime\": \"2026-10-20T06:58:30.195798059Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\n correlation ID: dbee8485-16c4-476a-9f69-1fb4ac6725f8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x218ff96e360\u003e: \n failed to create HCP cluster np-dg-minor-grlgd9: failed waiting for cluster=\"np-dg-minor-grlgd9\" in resourcegroup=\"rg-np-dg-minor-grlgd9-gr8jq47glbkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\",\n \"name\": \"648800c4-40f0-4f5e-9d03-bc80720958e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T06:29:28.955511783Z\",\n \"endTime\": \"2026-10-20T06:58:30.195798059Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/648800c4-40f0-4f5e-9d03-bc80720958e1\n correlation ID: dbee8485-16c4-476a-9f69-1fb4ac6725f8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dd26d6dc2ea514cbf71f866c00ebbb6f51ac812cccb061d19882178b33dcceb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"d3bd76292fc811dd516a618423d6ac72b982885c35039b4005f631a77c12b6fd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-jq9xq7j5pzr6/cluster-candidate-5-0-bpswhs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2bac6648c280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bpswhs: failed waiting for cluster=\"cluster-candidate-5-0-bpswhs\" in resourcegroup=\"rg-candidate-5-0-jq9xq7j5pzr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\",\n \"name\": \"9237519a-5ba6-44c7-884a-5549b504b6c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:21.964445806Z\",\n \"endTime\": \"2026-10-20T06:03:30.678161084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\n correlation ID: bacfd228-883e-4654-bffc-2565399b10e4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-jq9xq7j5pzr6/cluster-candidate-5-0-bpswhs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2bac6648c280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bpswhs: failed waiting for cluster=\"cluster-candidate-5-0-bpswhs\" in resourcegroup=\"rg-candidate-5-0-jq9xq7j5pzr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\",\n \"name\": \"9237519a-5ba6-44c7-884a-5549b504b6c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:34:21.964445806Z\",\n \"endTime\": \"2026-10-20T06:03:30.678161084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9237519a-5ba6-44c7-884a-5549b504b6c2\n correlation ID: bacfd228-883e-4654-bffc-2565399b10e4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ddb02547c344c8a8b304b70e08c4b5f5b656b1e4294b87a7f3d7d00a1d28d98a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"7626336d8319bd1b9484bd3d83ed4edd851ca8da911ba5db554a4e3dc064cc49","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d94cc015dc95597b28dc0d7b9906b72de1a3ac50e629b544d55f9dc2f683d21d","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"f2a6de6069481348c954465f96f9889461d7c1a41e212ea48553013eab443310","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"0aa6f50c96c553d375cc98020a5cade4e8279d752da9f9d38ebe012a62916580","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-xf65fl\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x10e087138a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-xf65fl: failed waiting for cluster=\"cp-ystream-upgrade-5-0-xf65fl\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-gtsv9z8h4wtm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\",\n \"name\": \"c1a64b0d-1df0-4545-857c-0372d97bd54a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:42.288189814Z\",\n \"endTime\": \"2026-10-20T06:05:50.682275366Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\n correlation ID: 6df68f18-7930-409b-8572-0ac1ed5cabfa\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-xf65fl\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x10e087138a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-xf65fl: failed waiting for cluster=\"cp-ystream-upgrade-5-0-xf65fl\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-gtsv9z8h4wtm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\",\n \"name\": \"c1a64b0d-1df0-4545-857c-0372d97bd54a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:36:42.288189814Z\",\n \"endTime\": \"2026-10-20T06:05:50.682275366Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c1a64b0d-1df0-4545-857c-0372d97bd54a\n correlation ID: 6df68f18-7930-409b-8572-0ac1ed5cabfa\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f6671e56ee775b3228188779dc8fcdf3fdcecae6fec76a32a3d1ec399c78d525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"2fa0ebf1230c4c554ceca7b28f0b25d8635d39b3dc24d3b5bcf7d59bec5783e5","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3989e2b60bd0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\",\n \"name\": \"d7404ade-6038-4055-bb7d-b4cfba1ff9a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:47:22.471947054Z\",\n \"endTime\": \"2026-10-20T05:16:25.547638023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\n correlation ID: 4d7866b9-df84-4478-8061-de7fdcadeeeb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3989e2b60bd0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\",\n \"name\": \"d7404ade-6038-4055-bb7d-b4cfba1ff9a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:47:22.471947054Z\",\n \"endTime\": \"2026-10-20T05:16:25.547638023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7404ade-6038-4055-bb7d-b4cfba1ff9a8\n correlation ID: 4d7866b9-df84-4478-8061-de7fdcadeeeb\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fab2ea54f8062fb11d176414945a75ef85e2e30c464dc7520d0692db7b2ef991","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"f15f317e2e6210638afaa7300c8e508fafa4ad2b0933a52186de02cfd00722c3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bg8pgz\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e13a41e4d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-bg8pgz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-bg8pgz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qx6srqc7dgqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\",\n \"name\": \"6f0858d6-2823-4181-8239-df67a965eec8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:34.606289117Z\",\n \"endTime\": \"2026-10-20T06:06:40.527532891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\n correlation ID: 28808a78-bbcd-42c7-8275-7ecbabeecf46\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bg8pgz\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e13a41e4d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-bg8pgz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-bg8pgz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qx6srqc7dgqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\",\n \"name\": \"6f0858d6-2823-4181-8239-df67a965eec8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:37:34.606289117Z\",\n \"endTime\": \"2026-10-20T06:06:40.527532891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f0858d6-2823-4181-8239-df67a965eec8\n correlation ID: 28808a78-bbcd-42c7-8275-7ecbabeecf46\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fb239a9abdd619f451c281d821ecd8bdbf3bc76c0499135d0aad8e49f4e8b35f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"f808c0f2ddbe5c78efd2332824a41685aaf34fa8d76930e4175ec5a4471d3a25","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b0e35a98ba0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xnrwkcptq2d4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\",\n \"name\": \"886255e6-39f4-410a-a5f7-92ac05a1f911\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:39:12.847857574Z\",\n \"endTime\": \"2026-10-20T06:08:20.525136392Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\n correlation ID: 9848e6bc-6d40-43e0-ae1b-5ee6f95fa4cc\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b0e35a98ba0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xnrwkcptq2d4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\",\n \"name\": \"886255e6-39f4-410a-a5f7-92ac05a1f911\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:39:12.847857574Z\",\n \"endTime\": \"2026-10-20T06:08:20.525136392Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/886255e6-39f4-410a-a5f7-92ac05a1f911\n correlation ID: 9848e6bc-6d40-43e0-ae1b-5ee6f95fa4cc\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fd5e7b8f34882e7c8edd0f74dda050e947c1970b5de5e1bc48cc7cd425e30363","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"047bd28e96ccd6a342a286dd98068aac1cfafaf02a11ef4a9793b341f9552f26","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x106428050380\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-85x64ntjvwjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\",\n \"name\": \"65962c7a-48f0-4bdf-835e-094037d9bd68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:06.594464666Z\",\n \"endTime\": \"2026-10-20T05:17:15.387268555Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n correlation ID: 64f4406b-d45b-4b7b-8f16-7904d655ca01\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x106428050380\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-85x64ntjvwjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\",\n \"name\": \"65962c7a-48f0-4bdf-835e-094037d9bd68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T04:48:06.594464666Z\",\n \"endTime\": \"2026-10-20T05:17:15.387268555Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n correlation ID: 64f4406b-d45b-4b7b-8f16-7904d655ca01\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fe609563c8e7d960cb8cd6aa86b29d14e4c9d657433879343b1a99ba546dc178","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","failed_at":"e2e","signature_id":"b3eca93c382c50fbc848769cca34fabfe3010f419203c0aaad7208643b60b483","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-fkb24v\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x4739e1a8f40\u003e: \n failed to create HCP cluster cluster-zstream-4-23-fkb24v: failed waiting for cluster=\"cluster-zstream-4-23-fkb24v\" in resourcegroup=\"rg-zstream-upgrade-4-23-79dbfbgzrcv7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\",\n \"name\": \"326cb528-9348-4889-baf2-110724ebc82a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:35:30.790310321Z\",\n \"endTime\": \"2026-10-20T06:04:30.51618977Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\n correlation ID: db0409d1-ff10-4a93-a0c7-9a65a61a2a04\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-fkb24v\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x4739e1a8f40\u003e: \n failed to create HCP cluster cluster-zstream-4-23-fkb24v: failed waiting for cluster=\"cluster-zstream-4-23-fkb24v\" in resourcegroup=\"rg-zstream-upgrade-4-23-79dbfbgzrcv7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\",\n \"name\": \"326cb528-9348-4889-baf2-110724ebc82a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T05:35:30.790310321Z\",\n \"endTime\": \"2026-10-20T06:04:30.51618977Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/326cb528-9348-4889-baf2-110724ebc82a\n correlation ID: db0409d1-ff10-4a93-a0c7-9a65a61a2a04\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":61,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","18083b0da87c5f910711dbd500417dc67679d796a27ea6259b1d8bd9fe4e29dd","18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","600ad974be9c7777150a3432faf4397d3733a0f1f567bf0b6ee9a4202295378e","60b366d51c10b5b9b70cb40285af4daf6406290c1d165863a793e90acfa07ec5","6ecbe2fdf10383ad3e6b4b9603afc5bfad8be6ed12177f6a3782005a280d5e7a","8bfd122bfb355a24d92287df3269e0a7f0e6a9d4d45c80a03f8103457622a6b5","94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","a4fad994d0f98f503f348d9fe7df4759af2e675b2b5dfd38e9ccf9fd05ba39ea","a841661ff321264fc6d63199430ded1478aee9bbad0f27351d0c98a58e041b68","bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","d94cc015dc95597b28dc0d7b9906b72de1a3ac50e629b544d55f9dc2f683d21d","e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c"],"matched_occurrences":61,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112363205513711616","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-20T02:00:08Z","started_at":"2026-10-20T02:00:08Z","completed_at":"2026-10-20T04:00:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T04:15:48.812220212Z","timing_metadata_checked_at":"2026-10-20T04:15:48.812220212Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T04:15:48.8159765Z","region_metadata_checked_at":"2026-10-20T04:15:48.8159765Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6f4c90ffa88a70cdb3e42720ea48a57f7514f90e6a32166c8b15f743a52fc9dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112363205513711616","occurred_at":"2026-10-20T02:00:08Z","failed_at":"e2e","signature_id":"26c4c256a226d294c012f0b9e34ff72385f3012bfedc1dac6ca0a79106b9ef85","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x3fd8b58f89c0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-r7w7ff\" in resourcegroup=\"rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\",\n \"name\": \"392e4864-c483-4741-bf7f-aafe39a6a315\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T03:14:53.109279817Z\",\n \"endTime\": \"2026-10-20T03:33:59.378971686Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n correlation ID: 93f4ee4e-2929-477b-b173-ef81cb3abee4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x3fd8b58f89c0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-r7w7ff\" in resourcegroup=\"rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\",\n \"name\": \"392e4864-c483-4741-bf7f-aafe39a6a315\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T03:14:53.109279817Z\",\n \"endTime\": \"2026-10-20T03:33:59.378971686Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n correlation ID: 93f4ee4e-2929-477b-b173-ef81cb3abee4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}