{"meta":{"start_date":"2026-10-19","end_date":"2026-10-25","start_at":"2026-10-19T00:00:00Z","end_at":"2026-10-26T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T01:47:18Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":59,"failed_runs":25,"raw_occurrences":244,"matched_occurrences":241,"runs_affected":22},"rows":[{"environment":"prod","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":21,"runs_affected":5,"failed_runs":5,"run_impact_percent":8.474576271186441,"anchor_occurrences":21,"after_last_push_count":26,"trend_counts":[2,1,0,0,5,10,5,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-18T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x9f6ae185170\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x9f6ae185170\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e14e7538290\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e14e7538290\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3184654d9920\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3184654d9920\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"33db1c3a753f86de5437aeea4ce94aea50eff6efcc9e24481fd3b5a5db91355d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"3f2b2137f83a3269a387de589f4ec90449d00621fe948040b817efb69653752e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"6464f6e3e62a79a6ecad40d9da5fa9f7a3ca4e1c274f987a91f46ad0ef5805f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"6aad85da11b23d51d01d7048ab2fcb7dd17e96a24595d2c070d12b0e2654fa14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"8d94601ad9e8035e94766bfdd3ff32bdad6a27008af0242092dd096fe5ae318d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"a9d0787a49a9434b575f780199c840b15cab7a7f5c8097f81f3e40e8e6452e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"bd72849f8bfb61f3950e07a620fb8b9bddc49d0f09058e4f6ff7f7eef02627f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"c3d8588a7330141641207b393ef0b056b3f565ec62f5715a57756d5035d5fa24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"e444cd22aa468d73f20a8d2ff0e73fd525cfd2b1f33d4eab9b629bfe695606fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"f8ad7afc2b67ed6f03b36c92fec6e7ed4f59bd175316c494010190cf2fe6a1df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"083741f16af6bde01a15c32f163b36e9e075c1b8a96f107ca16e1a1830aa7995","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"2d6ae229654176a356f5c870d83add62836595109dbb7a96870fa8c8c6a4e33b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"4a987599543343a0a61051617564a6330ab4d61da2e4fe37b0c8507e51dec2dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8ae7224beef582a60fc95922521582cea88650e7efa805e02a16af12f707e016","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"951e3efbb76f026e0f9c0427659ee8ef966b4d7a1594708f3b87a3b80dade019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"28f82d6598d8560878485b881cedade2766f6f99a7665f52a5642ec9863e8f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"554520898e3bf65aa33d08c67763b5d08098c2dea88239fbc79acfa28b30250c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"91144e10f499ef01ce91bc27b79924e73fc51fedcf3381a2242dc3972585a568","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"ad2974ff484501a74259c346bde4f86a21cbaef053d5d94585993a7186adc875","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"d6194b621062ac0dc7417d99a6662052b73f0727482224784de827ab3d686809","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112000853891092480","occurred_at":"2026-10-19T02:00:17Z","signature_id":"f192d550b7db929fb99903672f7a1a7817ae4c28de44c9612de5fab5e8cb2747","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":19,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":19,"after_last_push_count":19,"trend_counts":[0,0,12,0,7,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00e889108d18b2602a9701e81c628e0117c6af2ac6506f48e3dc6377714f2dec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3b522484ec379b7598bbd268e9f04bacd0edc777033a068970a7c75bb7fd3ebc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"51ee050adec069064368b692e2e9f92789ea23eb4ea27242862d06944731e6a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"93b4242fbca78a6804343fa0c780a64cee3b6f7f909e539aa95b885d3208f48b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a705768924b8c91914dfee887fd041226da8b0d2a7c49c71af1ef216422bf744","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"af774aac99c529ad98d882b83b9a1e38d054f96b4ba76e020f5318718f9b0f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d28602bc01604b20a4fd1810051138a3b4d9daa151d24a925a551647d550c3c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"2eb8f17d8e8fdf3ef65026f596d61ddd2edf43214a5d339daac7727d062002fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"2fa0ebf1230c4c554ceca7b28f0b25d8635d39b3dc24d3b5bcf7d59bec5783e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"49bc5863be8707ea32e900c899a764b317035418945da414b2fd1bb27cbd6874","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"4ecbeef13a630701a21a74ce0404abcccf3cb96ab3a6bed45186772d392d1cbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"859ea0d5c5bc8abfe7fd4ff75ecbc3fb7c86fc5e4d3a9f2febaf8197f05ec208","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"928184f02b9d110413b220f12fab11223a87fd1749e66886968a83d4f750b438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"a5bc08bcb1e4e6dcd7e514e3424f3eb0c78f0cbb642a2952ccefe4b2b4c6b3e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"ac10ce2cee387f866229f3daf6fd76c6a13c92110bcadbc1ee7e47af615dc842","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"b821b9ea697aca4b87f94938289d6cb621ff01fdd2288c0cec68b2091b62a01d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d3de213c99043332e16e77edea115f0ad9cddb7a0829c21a9a186e31e21883c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d53c47891282a5e167e06922b6c9850dc9d2bd94d5c1cd79f7cd49d1e12a8731","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d8b29925f38a23319a5e29ac21fd486c6701e6e44b71c79797bb3944c415017e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":19,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":19,"after_last_push_count":23,"trend_counts":[0,0,9,0,10,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-15T06:27:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00c7da51be8e89ce15ce8e10590e25d85b2282252d307b006276577d947b9e86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"01479cc8260099169681c3c9aac7e1601c47f3919a6c41c57610a9af6d7dfa91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"2ca8d85c3dc2b47b78c6a24945b274f76b13627c62c8a5c01d3ebc927fad0021","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"541341a5ff6a403f156c133d1168366027c4c77d725975906770d24cc82a1c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8443649e9083d1ab2db3cf3952a87eb977678118008a762dcd368311d2aaac93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b311e9b9b11e8b11ec8d0f86f4d29406e7d83165875e0863fb9281515f0c7523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ba521c2c19919a5caa070da355f36879cb81eb48d013d05ff609671940da6a95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"c0bc7ce2f8badc2267d7a65ad1002f4d46160b2c3b4087c1135728e23c2b816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e4e0808ae7571f998df5b39264b674c05cb041138f2cb2532613af052179af56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f7d5f9aaa39bc4f25aca130a942f0670850a3d28be7139dc0a9df1e58b8e9577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"0aa6f50c96c553d375cc98020a5cade4e8279d752da9f9d38ebe012a62916580","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"0c0b1582bdefb57620af76df4308630b0656dcc4f4d034080e6eb8bbcee22919","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"0f0d911e3baacda1fb2ae94c5502b51bfe46c06febf46f973e63beec4d9e1ee8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"3faaa107fa90727c96a34d72451750f7291409b311a0a4a67821b868f59cbab2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"4c53253f9e54053e65f1d9f31d1f7c897f3ed64b8cd91d2d5a475876c4218f25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"5953dd256eb587494cc84cbe3d24465e66cade2a891fa9dea6669db6066c85b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"9e1ec5a9d28e9a47aa98f55666f7138cef8b72cca132d14571662467161fdedc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"b3eca93c382c50fbc848769cca34fabfe3010f419203c0aaad7208643b60b483","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"b6c9b1a6dfebf2ea1ce6eab0f805c9997bb911c59710974cf5cf2a393c6b58df","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":17,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.779661016949152,"anchor_occurrences":17,"after_last_push_count":17,"trend_counts":[0,0,0,0,5,7,5,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-vsfrhd\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b75911f3b00\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x1b75911f3ae0\u003e{\n            observation: \u003c*errors.errorString | 0x1b7593506210\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1b75911f3a60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-vsfrhd\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b75911f3b00\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x1b75911f3ae0\u003e{\n            observation: \u003c*errors.errorString | 0x1b7593506210\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1b75911f3a60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-hvm2zb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a2d820f0740\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x3a2d820f0700\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x3a2d820f06c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x3a2d82526860\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-hvm2zb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a2d820f0740\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x3a2d820f0700\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x3a2d820f06c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x3a2d82526860\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-m5lclb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d085988600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x12d085c685e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-m5lclb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d085988600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x12d085c685e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"48bcb095dab95d4300f51495f479877c48fae6ac937eca31ce9ee317ffaa9c61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"b1dfe79b23dde219591e24160c9becc4dd2e4815fc615b83f459dd71654c29ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"c38d06858d691d9e4c16513cdbf71adc44dfc0e2e5bba3c53da574b0cb0fcaa2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"d12f7ad296c63d2db8d0091c07a0e4b76c50e119a987979054a17bce6c78d0bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"f2369be4ef1db102806ff130f76a044fc5b4c806e03890b5fd3f3eccd39596a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"5124c660676e607323f38b7ee2b1cca5949648ceecce1866b6241aed16b10219","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"f9b8e244e278e564970161a86e6652a63f58015bd2e7dc85de719e5bba6ef252","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"13d3f7c24d1951f7eccaeb591f09a7ad27b72a1f0d3c63a0a590154ec1d02b2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"59e0905d51abc3f260e60eaccaab70b634685836f78308d19447082523bd7935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"5dc7704bcbce0f4a7f9f81907bd6a202126d16c8173b589ea3e9a7efd501998e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8adea4c35fe3f9117bdf109bb92ed50cc3312cf016d9f42e8bb8d7d02a60e613","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8b45e8c93fa73392b7a5435ed12dc6c9c04d541483629a0f2c963d83febedd8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"11a1ea4698faced47725bf493b4aacc2124922e05d2aed8e74789e91de7a5717","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"60d8736178a39e3a5b4fef671fa2ef1330288eda4204ffbc60e31862e2471d95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"7d7a4bc924021f4fa04d801c41a3ac27180abe1576c70ed76bd677cebe7f2a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"a0af21048f0851309014ba5e3dc0d0f72ace62a5a58aa35d7e2b48b85ae30c13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"e51db32a50ceffa0ea6a335e06c32c375c606cf3ce2f73a2c6b002fa3305aea2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":5.084745762711864,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,5,0,7],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-5gtk26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x187dce020660\u003e: \n    DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x187dce1941c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-5gtk26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x187dce020660\u003e: \n    DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x187dce1941c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33aec8dca6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33aec9a24040\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33aec8dca6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33aec9a24040\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaa72ff3eb40\u003e: \n    DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xaa72fd7c9e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaa72ff3eb40\u003e: \n    DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xaa72fd7c9e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"254849cbf5d8311752c99c9248a16fa04d7f6f2913e5fb83b7809a02a5a35916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"5eca3309ccd3e721e36f955090b7fbc956541d0a385657c2f9a6b84b63b4ed05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"63b9c0750fd00f7e0bfbdd70f47df6c7c8f5e5ec84bfe85b68318083b3ea2e5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"7d2b6c735bdffa857d31b502d0f673ff158743f511cc94c645b379eec32fe280","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"9d4340c3d5dd7a61b94734bf26b15b1bc36bf39a7633f96f82d30834ab5f4c8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"daa48f2da7fe25cf04788befd5b4892e4a30494e022bdb921224e7051f37fd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"e18d1395402ed17099a1459b75389ff956491b242d6bc7519d732c7e1751ed7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"0b5d6aa96dc387e24884d930781d5cd7d6557a061c28ab9733a3d3b1c2b4c58a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"dd7ef8065ee1042cfa5c3e7a73f09180f1300ed5a4e62e2f76b03643b5c75908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"1bd3c4c4ad3a9bdeb00d2c5694e282e664d811eb31be8ef577772f8ad9b54e05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"8e5db3d65c7b7608ee7e78aa4ee79fca7ba6f2e43840a7bfc37e8268baf454dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"aaeb63305e9f896148e97071cab84607802b6d3dc40093f2e77fe61a4284ba60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":11,"after_last_push_count":11,"trend_counts":[0,0,2,0,9,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"105a43b4bf54437760ddaec80d7274c76e0880fdd04198d3317c96b1e41e0747","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"179000d9fd7136a36d03dbd3bfaa3c8edbf92638e8ad95333390e6c1eadcd7dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"19248e56dbbe622fc0ab707175608011b12ef937b74bee06e900da1ff17799d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7532d5417c94c768b19029dc4b5188f6d4f7b0525e268ed507fdd211a7691731","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"78bb0fd5dc8b2957b03be693a8b15a24b0624bf8afc1be3e13738a6518bfff22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"89edbfd547763da1be2ecb5ac3a37c254a734ffe88e2c8157270658bdc6cd116","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cb6c62fc5bd1672b019609253e5970cf0be960ae29f59adef3f3e3018a703317","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e50814639bbed4fd159acc63cc76eb3b3ea44eedc3c0972c63ffedfeeb1d6b35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fa2dafb46bcd964c71b90a558a536dd7120ddbfa5212144072e39dc9ca2ed0c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"3a4bdebb5a54840d29dd533497f38dba7690806bfcea4d2557d63b7ca4439497","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"e2840043f5fd0a2eda7b3ca11b292f1b1f1e365717f405d42a32dcb35f6fd4c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":11,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":11,"after_last_push_count":12,"trend_counts":[0,0,2,0,9,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-12T13:01:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"012587d7d4630c81e42093fd6ab2c1257951217e2525d7784fd89a2e89112676","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"1e540a6273c70580b8d0a813a5ac2b3184c764a6be285464f22624d92e6da8b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3869c78574390577fcc299556aef741fbec4d0cca2bc649c0c2a2e1cd5f36f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3dd0847594fe9934ecaef404cfd5b2e0d30bdd83e3bdd0629d37df342bb8fe04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7a1cec7cd36f34b86775ec4e998ac79d4f3902ec309e0df7dd447a697e1ef501","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"884359bc37b7b0f624f652c6e3ab6489f45271d6a5f2eaf45cf5d681cd51b93c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"940d1146406ccc6a3297d550ec44a0b5d1a43287589036549f13183fd87e514e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d15e0ee9a765947a51994971a21dbf7f7f640ff3a8e13f8d501e97316301c1e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f052c83930df14da60956d205c94d7b6903bdbd091c343818e9c8edabac6c7c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"a464e881b3b5384903bbf8b5cba5cbec43270a9c169c5298ecd2ca63b6863f35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"e9ce23b3d7e9d97edaca0e32d8e720245c2455d9d7bccc974ec7f9ca371f3bf4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":10,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":10,"after_last_push_count":90,"trend_counts":[5,5,5,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-18T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x16962b1f6f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 9ef8b68a-bcc4-4e41-9cf0-46d6272fce74\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x16962b1f6f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-m6l655w65gt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 9ef8b68a-bcc4-4e41-9cf0-46d6272fce74\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xaa2ec6a3830\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 11208f71-8029-42d4-aa43-9f639823791f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xaa2ec6a3830\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sm6hpjzmbhjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 11208f71-8029-42d4-aa43-9f639823791f\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x32eaa59cc210\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 37d0ceb3-cb28-4457-98b9-e7ba4d280731\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x32eaa59cc210\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-r92b5qqc8544/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 37d0ceb3-cb28-4457-98b9-e7ba4d280731\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","signature_id":"51afcfdde45a0aa3ec6d3b1b3015660ec4a7a6ce7dbf663ce5a3a178d9ff23ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","signature_id":"8ad7e88444f4ffe5bf5faead1e295bf1c4daae12a4cbae9a31a7d7dd1ffe08b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","signature_id":"94536779431dbad340de6d98c947080d665396d79fa2590db0ca47b199b6f768","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","signature_id":"9dc909a5d64b01d035d949a7efc84e5c1444d43ea7fa70c67f943609bf582bd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112680298507407360","occurred_at":"2026-10-20T23:00:09Z","signature_id":"cb24ba12df08b46fc09a4f838a3aa039e729f29b3406c143bedd14694edad6e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"19280e1cb89eff1510005486a6db480a00b91a3d6803adf465f6bd5650b1646e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"3dd9e65e4d9743826d1d1be249e0e445d0d9d7c6e4b726244d67122f09393ca3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"4ad20b2e6c8e87a355ad5f1df094851da560505023ea56df0a4565a242c3bec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"c4fea1d46473b4b5e7836ecb05c8776492bef6a0adab3d1fd5134cc124a5972a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"d929e7fb07063ac4062e24c44fff5d60325dd32e5d68a6c1facfbf05cae9a758","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,6,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-zjk46l with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de0e4b15080\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zjk46l: failed waiting for cluster=\"np-version-upgrade-cluster-zjk46l\" in resourcegroup=\"rg-np-version-upgrade-zjk46l-52b2swhcrs62\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\",\n      \"name\": \"7572214c-58bb-42db-9c41-46711f0d39e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:21:20.599092026Z\",\n      \"endTime\": \"2026-10-20T06:50:20.225694377Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, dns-operator, control-plane-pki-operator, kube-scheduler, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, community-operators-catalog, cluster-network-operator, packageserver, kube-controller-manager, openshift-controller-manager, catalog-operator, olm-collect-profiles, capi-provider, olm-operator, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, certified-operators-catalog, machine-approver, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n            correlation ID: 2a2f8f2f-1289-48c9-97d5-1f5137565aa0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-zjk46l with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de0e4b15080\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zjk46l: failed waiting for cluster=\"np-version-upgrade-cluster-zjk46l\" in resourcegroup=\"rg-np-version-upgrade-zjk46l-52b2swhcrs62\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\",\n      \"name\": \"7572214c-58bb-42db-9c41-46711f0d39e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:21:20.599092026Z\",\n      \"endTime\": \"2026-10-20T06:50:20.225694377Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, dns-operator, control-plane-pki-operator, kube-scheduler, cluster-storage-operator, csi-snapshot-controller-operator, redhat-operators-catalog, community-operators-catalog, cluster-network-operator, packageserver, kube-controller-manager, openshift-controller-manager, catalog-operator, olm-collect-profiles, capi-provider, olm-operator, hosted-cluster-config-operator, ingress-operator, konnectivity-agent, certified-operators-catalog, machine-approver, openshift-apiserver, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7572214c-58bb-42db-9c41-46711f0d39e6\n            correlation ID: 2a2f8f2f-1289-48c9-97d5-1f5137565aa0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96c75f1ee40922482fa619e484710e1e93eb36cf6eaf5f78027744f7acdd09b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d2d766d5aec5f079fb45688d592a83dedffde25333099326b9f7f5f8e100d8f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"010478f08c7ee8fb5fb2617ab302b6ce848de473bb39c93c97f69270deb122a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"0719c23731937943a86a2b3aa9a99194c166c5e5b9d50721b270d5435f98088d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"136f3e529c41bdef3473933d2081faf92c073f86ca46c938c2727073e0dee688","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"290c508f7f2e85c3460635eae7ea41f5b7780ce968d6ca80abb77cd3bee95036","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"30230279c2e3510d994380632c7a574f5e6af12cc1a0c4498c3b30dd95687f47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"5a163e5e0a83014f555d69d77aa75af4bf1868419228ae0ae4dfd4a51baa6b7c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,4,0,3,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"0e6ceee3cd3e93558b27b65ed689870c63b26dd3bc15ef1fe3541ac45579879e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"6d20fac6b2c664e06dd79b23d104b0707d6bcfcfba1dad3f3a5feeea9f41e97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"74f59f951658f16f8d54e8163c072eeb6209baea5d1369c72844e0d3da80a25d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"39cb4b853fb39042d89ad859200027cc772c814433d47a8e1f5499bd7bdff16c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"914b343a149d69ce3d4ecefdfb503cd55563fec6ce1e76edecf68478fb484783","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"ed4d2ea5ea2e59de470ab104b735900451489e3a699014d5642dd9a2cd16beef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"f63a6ebbc5501e9879294a8f133573498b6e673dcf7325d2092848e0d64f6ab8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,3,0,4,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"30b3256251eebbc6ad2f2d21745e8e3e6c242862e551efcc5662d3dab722db4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"37af00141384f00be88a064e4ee9337863ec4649e69343bc6f3df05230fa74a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"610533cebef7feda8d458dc8e967f9408265cfa9d327e5638860a65b8ee27823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b542846af6df57de6bbea4872c890fa54411aeb1201b4a017c6a8bfb54ce2c21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"3e55f008c63964d562d24005c7da675a204730845c588ff0e4ec44a7826514f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"55d3cab10d4262bc680cdea2d0125d8ba2c43778388a8535e053d3394c2f29ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d3bd76292fc811dd516a618423d6ac72b982885c35039b4005f631a77c12b6fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,7,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"06ff6d227a4a567e0691243be5eccb0caeeaaa315bcc42dc0fea399f15c10736","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"2cc2591b74b05456e73089bc9b36c0f1f7eee60669fc135f3ce15bfec52d2937","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"4fa8b7fc464071338bdd4561210059a3e2c75c44318bf3c18a41e27d54116b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6f30c7a8231594aeb46f3d54216c8a117b477376ff72a9a27924ceba4f803e6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"aca4445f1d5148a20ae5983cba5de62c36b6c0a7b69db5f707a5e755bb5182a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"dd0b0b8df7b2166bc054ede8f1d58b688dc0dac527d6b2fd66ecb6ab9cb8ddc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fe77c534f2d11553fbb56bdbdd25fa10e6b63e20abc56954df520c44f8dde521","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,7,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"022548c08c7f3d00dcc2c6ccfbb90c7621484b0f6cc92b83282d239b2bf97fec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"0705e7ce0972714a7b11f80b2c8aff85f61cdc6febe2637f5a146fe039b1f537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"3fb0e1297078af590b2d19465e2eb4a0832651168d99f0176c33e483a935dcbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6d820660123e1f2b46bada2acc37a3d7025031ef13070c0759df5cdf0c7b2de4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"b8acb5f0bafdb5a3735bb27828a6b4f3c5003359238914449eaa893ab955598b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"e4a1953e3872ab73515a0438ce007de9e2def00ae922c3087992e0fce1014e4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fd25ada7f7c17331da353bb415e7abc6bfe24733b76e6dae5911dba3aa007638","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.779661016949152,"anchor_occurrences":6,"after_last_push_count":12,"trend_counts":[0,2,0,0,0,1,2,1],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x5c10dd80030\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x5c10d35ddc0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10d35dda0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10db35540\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x5c10dd88020\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10dd88000\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10dbae320\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x5c10dd80030\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x5c10d35ddc0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10d35dda0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10db35540\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x5c10dd88020\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10dd88000\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10dbae320\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114492215458795520","occurred_at":"2026-10-25T23:00:04Z","signature_id":"ead8c420db783411710bfdc18c3ac594eab0616d6007cfdac94e624578b57afc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"4a296b8184626c9cf0da738c096b9a7efd69214488b2d8d7c86178f33312d094","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"f4dbd9fc7f76d3d491864efc557c3c22f1a4295a0221ba7f6fc875ded0a8dd16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"489c87a4b234518c7dfe6b37a3d19c5c6af589cc404fb24f4b84bb4855abdbd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"1c4887438713ccab4897cdb5741f70ac75c4f14a3ce6a8eeb00ca2af6d569929","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2112317906506223616","occurred_at":"2026-10-19T23:00:08Z","signature_id":"dc5cfcf5867163d3d4c94058abd30ec3a2ca855916a5104b85ef6a075dc7c9a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,6,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n    DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n    DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n    DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n    DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"a23174899568b6c480cde29c16f107e2df7fb592b2576c4598c4dd3b3ed64763","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"d9fbf23376585cdfaf80950f7a3f5e8696d3b0c22688297d50b6821493fe3062","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"eeff509555b3bd3bab3c3359799b2c2e597d7c59acb0e88f24bfbf61670e37e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"fffc6a8a7c348c164995910dd1c108865b80e864e9a58f247d4be169e8a399d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"476aa5c24e557ad6a577878e82a38cad972775741e8c95bde27dcf8780a1adf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"baa9afa3bcbcd0c44804b70a02b3154a77e55643457f2cbdcc513051780298c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,4,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x253bb877e5a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nng4k2kf29bc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\",\n      \"name\": \"b46a45a4-dcba-413a-9b74-f904cf76e493\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:29:17.820195399Z\",\n      \"endTime\": \"2026-10-20T06:58:20.185763239Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n            correlation ID: a247ec97-7f49-437d-bb0b-6d5fc996e055\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x253bb877e5a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nng4k2kf29bc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\",\n      \"name\": \"b46a45a4-dcba-413a-9b74-f904cf76e493\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:29:17.820195399Z\",\n      \"endTime\": \"2026-10-20T06:58:20.185763239Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b46a45a4-dcba-413a-9b74-f904cf76e493\n            correlation ID: a247ec97-7f49-437d-bb0b-6d5fc996e055\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"222ffc9c563b0b4467dc853c34c77c4694cb2ce41c4c51c01cfbc0965f89f195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3918b5053e2c1b6926508abb7cd5f09e8c960484f7c759d88c7b0673e703991d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"71922cf41f134281ebadb3bdb4fb82e42493d9860264a253b22bc5350e92e1c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"b63af5629d6d99f87cae38122036785f946e70231cea7b91bbaea3046bb3156c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"f15f317e2e6210638afaa7300c8e508fafa4ad2b0933a52186de02cfd00722c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"ffaa5c86bbf96c4d215750cac34aa61535716a53507e4cbfbc11c0b6cf39ba38","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,3,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x106428050380\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-85x64ntjvwjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\",\n      \"name\": \"65962c7a-48f0-4bdf-835e-094037d9bd68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:48:06.594464666Z\",\n      \"endTime\": \"2026-10-20T05:17:15.387268555Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n            correlation ID: 64f4406b-d45b-4b7b-8f16-7904d655ca01\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x106428050380\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-85x64ntjvwjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\",\n      \"name\": \"65962c7a-48f0-4bdf-835e-094037d9bd68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:48:06.594464666Z\",\n      \"endTime\": \"2026-10-20T05:17:15.387268555Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65962c7a-48f0-4bdf-835e-094037d9bd68\n            correlation ID: 64f4406b-d45b-4b7b-8f16-7904d655ca01\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"10bb0bdb748cd2f51cfdd05065854a330965cf2734479264377206588e187b5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8181e1d728fd2f9d65a27b1a6f385b120f5a9c516932134b128d158f80fe37e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"047bd28e96ccd6a342a286dd98068aac1cfafaf02a11ef4a9793b341f9552f26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"3b304d4ecdb8dc3eda09b0dd43c3474f5ca2580737f170b0f34deb2e5dd22d25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"f808c0f2ddbe5c78efd2332824a41685aaf34fa8d76930e4175ec5a4471d3a25","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,3,0,2],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x33a1f1315158\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x33a1f0cea780\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x33a1f0bd2f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x33a1f13e4360\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x33a1f1315158\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x33a1f0cea780\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x33a1f0bd2f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x33a1f13e4360\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xeeb7f5e0a68\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xeeb7f3b69e0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xeeb7f5137a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xeeb7f512040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xeeb7f5e0a68\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xeeb7f3b69e0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xeeb7f5137a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xeeb7f512040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"0ac27c2c2596a79cd4371dcfd3d01134184bca542ebdf146502aaa026ae80229","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"78fd5d6ed20f79e2553486e2f00d359fec4b08aeeed6379beabf7bf312ae9a76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"05baccf373c01bd6f7562dcbc3236b5780751cf9ec7eae7616fd70c9c8c3da1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"103697c5a8db15b41fc2a08d46ab15e59d6e4653e09a4e1f747b198e64c261d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"959d7136fa8fa3534927661a3d60094a7592b6d059d009358f71e15228fe76e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-2lr49","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":5.084745762711864,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,2,0,1],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d8dba328160\u003e: \n    DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d8dba3280e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d8dba328160\u003e: \n    DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d8dba3280e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"076cacfa7561e03f7335026846eac17d1438353985ba609f1c98efdf0e3de954","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"68fc0f1ea6cf0f44536f6ef401fc1e1215329535859894d52835b6c6b59f5866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"16e5dc89de6f908080887b19103bb8d46e6a6d30176df898a38a625fe63e5380","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a6864331b74f9520caaa01fb142d42bd934a1b337ffea6fcc51fd907a468b304","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d3dedfa37fc8f8f9d48c596be2117ea1ea85b98d4f8b0aba0451aa1f7bea81bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fd02807286b5b83c10835eef9428f86b7fe1936bd74115749fc73fc5111727ba","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-s5dcsw\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15e376eb4ba0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-s5dcsw: failed waiting for cluster=\"cluster-zstream-4-22-s5dcsw\" in resourcegroup=\"rg-zstream-upgrade-4-22-t6t94hh7j44m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\",\n      \"name\": \"08e66858-550b-405f-b39d-e72d4d3457b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:35:01.18923569Z\",\n      \"endTime\": \"2026-10-20T06:04:10.578239766Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, kube-storage-version-migrator, azure-cloud-controller-manager, packageserver, community-operators-catalog, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, openshift-controller-manager, olm-collect-profiles, kube-controller-manager, kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-operators-catalog, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n            correlation ID: feffceb9-72b0-45a3-8f5c-db4fbe3e6004\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-s5dcsw\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15e376eb4ba0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-s5dcsw: failed waiting for cluster=\"cluster-zstream-4-22-s5dcsw\" in resourcegroup=\"rg-zstream-upgrade-4-22-t6t94hh7j44m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\",\n      \"name\": \"08e66858-550b-405f-b39d-e72d4d3457b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:35:01.18923569Z\",\n      \"endTime\": \"2026-10-20T06:04:10.578239766Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, kube-storage-version-migrator, azure-cloud-controller-manager, packageserver, community-operators-catalog, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, openshift-controller-manager, olm-collect-profiles, kube-controller-manager, kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-operators-catalog, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/08e66858-550b-405f-b39d-e72d4d3457b0\n            correlation ID: feffceb9-72b0-45a3-8f5c-db4fbe3e6004\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ac79520c264a03cdae111d50487f33410c268ee3536a309598ebaf8fa5495141","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d706d1ccca667e86b44aee723bd6c4d7339497578c8a703fc65f912c293de8ea","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-15T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n      \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n            correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n      \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n            correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fd8b58f89c0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-r7w7ff\" in resourcegroup=\"rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\",\n      \"name\": \"392e4864-c483-4741-bf7f-aafe39a6a315\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T03:14:53.109279817Z\",\n      \"endTime\": \"2026-10-20T03:33:59.378971686Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n            correlation ID: 93f4ee4e-2929-477b-b173-ef81cb3abee4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fd8b58f89c0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-r7w7ff\" in resourcegroup=\"rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\",\n      \"name\": \"392e4864-c483-4741-bf7f-aafe39a6a315\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T03:14:53.109279817Z\",\n      \"endTime\": \"2026-10-20T03:33:59.378971686Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-42hc7 to exist\\nMachine p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7w7ff-gzcdhfsz9lf7-np-upgrade-r7-b213f862/providers/Microsoft.Compute/virtualMachines/p4u4z6x8s3x8m0b-npupgrade-4-21-wfvbp-pkjw5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/392e4864-c483-4741-bf7f-aafe39a6a315\n            correlation ID: 93f4ee4e-2929-477b-b173-ef81cb3abee4\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113640155058278400","occurred_at":"2026-10-23T14:34:17Z","signature_id":"83894b496f209ee26cbf0e4d1f1a178c6a6d6f9bdf0c14187a9b22ad01f74877","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112363205513711616","occurred_at":"2026-10-20T02:00:08Z","signature_id":"26c4c256a226d294c012f0b9e34ff72385f3012bfedc1dac6ca0a79106b9ef85","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"8a9ded4e992ead494e672f2c33210492690aab0b14e2de539e32782deff72275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cbeccff849e6332c34aba59d393fdbae4f3ec1001f41174ac4f54440f6912fdd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x471790ed8f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x4717a706200\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x471790ed8f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x4717a706200\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x397e13438ac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x397e134eabf0\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x397e13438ac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x397e134eabf0\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"928b1de9065772d3f2c7e201fb9ae101d2fb2df8e5b2cbd4ff7e82df961c73ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"bda22742bb4bcfe82726d83de21fb4e400993a95b279453234202a917cefb495","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3ce2ba89b8e9686083a83dadac7501168f4efe9a7eacff619946e50e9db1c762","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x1bbed2a5bb90\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x1bbed27f1680\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x1bbed27f1450\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x1bbed2a5bb90\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x1bbed27f1680\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x1bbed27f1450\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"2e48272f7c6962293e37e27585cc8a5c9247db1c6e77cd891e766919f7108086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"86a169f78b189ca410e85207ae5eedf317f888d81a7fbf1f8ae05a6c40c5927e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.389830508474576,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x3a0f21760840\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-m69f8px6srdw\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x3a0f2173b0c0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x3a0f21760840\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-m69f8px6srdw\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-m69f8px6srdw\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x3a0f2173b0c0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-m69f8px6srdw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f4c91ed9f592165df1c3c27e2fd22c4b93dc93ae321a35275a1f80ee00060e71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d756b1de5d0f247e4ca4acfc5768973e04b86c119eb06d43ef0dd144ba7446e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a841661ff321264fc6d63199430ded1478aee9bbad0f27351d0c98a58e041b68","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x87fabdfe6a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nb9lb62595s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n      \"name\": \"879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:20:18.481707716Z\",\n      \"endTime\": \"2026-10-20T06:49:20.225008169Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n            correlation ID: cf264c90-6764-45bb-a904-1799b612e538\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x87fabdfe6a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nb9lb62595s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n      \"name\": \"879b385a-2ae7-4f99-a707-d8f19a66cb12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:20:18.481707716Z\",\n      \"endTime\": \"2026-10-20T06:49:20.225008169Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/879b385a-2ae7-4f99-a707-d8f19a66cb12\n            correlation ID: cf264c90-6764-45bb-a904-1799b612e538\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2dc51d83d2c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dsxq6jxzmbtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\",\n      \"name\": \"30bbed47-9a91-40fe-b99b-caa97b286f39\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:24.928973066Z\",\n      \"endTime\": \"2026-10-20T05:15:25.520014214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n            correlation ID: d0e5fc28-aced-4621-9518-831e81cdc82c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2dc51d83d2c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dsxq6jxzmbtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\",\n      \"name\": \"30bbed47-9a91-40fe-b99b-caa97b286f39\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:24.928973066Z\",\n      \"endTime\": \"2026-10-20T05:15:25.520014214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30bbed47-9a91-40fe-b99b-caa97b286f39\n            correlation ID: d0e5fc28-aced-4621-9518-831e81cdc82c\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"5bed8bd505b275faee3a7c65055544105675a2bbfc9f487e68e44672dfb62806","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"5ce5e5f754031dd5e2b520b97795f008263e7e8ee14634915d4fd663395567e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a4fad994d0f98f503f348d9fe7df4759af2e675b2b5dfd38e9ccf9fd05ba39ea","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f03b61ccac0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xkdl5ldkwkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\",\n      \"name\": \"60150ecc-c0c8-4ddc-abff-d4417c838152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:17:57.832413824Z\",\n      \"endTime\": \"2026-10-20T06:47:00.220229717Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n            correlation ID: 9cce0d52-7c81-4f97-a269-e15c1e2fd419\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f03b61ccac0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xkdl5ldkwkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\",\n      \"name\": \"60150ecc-c0c8-4ddc-abff-d4417c838152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:17:57.832413824Z\",\n      \"endTime\": \"2026-10-20T06:47:00.220229717Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/60150ecc-c0c8-4ddc-abff-d4417c838152\n            correlation ID: 9cce0d52-7c81-4f97-a269-e15c1e2fd419\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3bda3528c680\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxsg6grsgkll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n      \"name\": \"2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:23:14.734977136Z\",\n      \"endTime\": \"2026-10-20T05:52:21.558247967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n            correlation ID: 2af21d6c-711a-4dc5-b3d3-22e3f2257e3e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3bda3528c680\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxsg6grsgkll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n      \"name\": \"2c866c88-017a-4ca1-a34c-aab419b47bd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:23:14.734977136Z\",\n      \"endTime\": \"2026-10-20T05:52:21.558247967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2c866c88-017a-4ca1-a34c-aab419b47bd0\n            correlation ID: 2af21d6c-711a-4dc5-b3d3-22e3f2257e3e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"9e3c71a0b8f48edf31f81234c87ded5a299025a3d5e7bd40495202e2fd2aa54e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d43e5e329e630584b37978546c0bf82691057e6c069848ea21832f0299660b4e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"43ca0ef5137db4c92687ca44af2e5269c14f5a14f38084e6f18c59c868e18577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"66fe9486ebbdab2264fdca019d93b069038bbe27a27a787cb2a4da95a08e65ec","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3ec3f52024a46a1f942ae3af19740a4993c313ed0bf4cec5b820ab716098ba7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96e526637370066fb982529359f8fb2c1357e11506ca6d9024d12db800540150","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-jn77rzqmfkjz/cluster-candidate-5-1-j65tbc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x332b74a98940\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-j65tbc: failed waiting for cluster=\"cluster-candidate-5-1-j65tbc\" in resourcegroup=\"rg-candidate-5-1-jn77rzqmfkjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\",\n      \"name\": \"be69220f-b32f-412c-9de4-3a1f140f5135\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:34:23.131093305Z\",\n      \"endTime\": \"2026-10-20T06:03:30.67205401Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 9m40s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n            correlation ID: 8aaf06ce-d046-4047-82a0-22fbcc3d8734\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-jn77rzqmfkjz/cluster-candidate-5-1-j65tbc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x332b74a98940\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-j65tbc: failed waiting for cluster=\"cluster-candidate-5-1-j65tbc\" in resourcegroup=\"rg-candidate-5-1-jn77rzqmfkjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\",\n      \"name\": \"be69220f-b32f-412c-9de4-3a1f140f5135\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:34:23.131093305Z\",\n      \"endTime\": \"2026-10-20T06:03:30.67205401Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 9m40s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/be69220f-b32f-412c-9de4-3a1f140f5135\n            correlation ID: 8aaf06ce-d046-4047-82a0-22fbcc3d8734\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xae91b2b14a0\u003e: \n    failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-5zjb4vwqmp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\",\n      \"name\": \"74351f12-7936-4804-bc6e-44e43566ba15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:37:02.298920957Z\",\n      \"endTime\": \"2026-10-20T06:06:10.689160757Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 13m18s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n            correlation ID: 139f95f2-0f80-424c-9b9b-7d135360bb2a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xae91b2b14a0\u003e: \n    failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-5zjb4vwqmp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\",\n      \"name\": \"74351f12-7936-4804-bc6e-44e43566ba15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:37:02.298920957Z\",\n      \"endTime\": \"2026-10-20T06:06:10.689160757Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 13m18s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/74351f12-7936-4804-bc6e-44e43566ba15\n            correlation ID: 139f95f2-0f80-424c-9b9b-7d135360bb2a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"037ab5604e242e87117bb5a1f7a62b9304d6f5396451277b9fc3608601ed2669","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"8cb5b55f31703889227570c650157a29009d93bae37355a94b6d3d03cda9c20b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"908166ee7c49cf07f1a2b5b18714d6e5f70255daa4c1d99ba634878b4c3b8017","failure_pattern":"\"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 307.279s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0x337cde092010\u003e: \n    E1021 23:30:50.528002    2868 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    {\n        s: \"E1021 23:30:50.528002    2868 memcache.go:381] \\\"Couldn't get current server API group list\\\" err=\\\"the server has asked for the client to provide credentials\\\"\",\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 307.279s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0x337cde092010\u003e: \n    E1021 23:30:50.528002    2868 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    {\n        s: \"E1021 23:30:50.528002    2868 memcache.go:381] \\\"Couldn't get current server API group list\\\" err=\\\"the server has asked for the client to provide credentials\\\"\",\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113042690441482240","occurred_at":"2026-10-21T23:00:10Z","signature_id":"c4eb20e93ae75373c59464f00b68f59434fe607f1d433e291947a1182f458028","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b286b88e4cdf2beb25fd0ed17c671477745b97698c63efca626d18c2b4af0e4","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n    \u003c[]string | len:2, cap:2\u003e: [\n        \"console (Available=False)\",\n        \"ingress (Available=False)\",\n    ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n    \u003c[]string | len:2, cap:2\u003e: [\n        \"console (Available=False)\",\n        \"ingress (Available=False)\",\n    ]\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"3bfc2a89887bfd392938b2d517fdbf090bf267eaca9f9ccad980b9907856ef39","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"169f70dba2351443907226ac5ccc1485fb8e510b011ab14b2d8af60b14150235","failure_pattern":"DNS for route host agnhost-e2e-sample-app-lpqxs","search_query":"DNS for route host agnhost-e2e-sample-app-lpqxs","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n    DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n    DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"cdfd5eadc24e0be0c150b2244b8232f0a89d3c7047aecd3074d3a83bba715ea6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"60b366d51c10b5b9b70cb40285af4daf6406290c1d165863a793e90acfa07ec5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9ffbe0a6720\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-c2tg56ndlf7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n      \"name\": \"862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:25.770566501Z\",\n      \"endTime\": \"2026-10-20T05:15:25.516223632Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-storage-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, catalog-operator, cluster-version-operator, ingress-operator, olm-operator, openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n            correlation ID: d433d7b9-c808-4842-80b1-be90eb0e3f4a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9ffbe0a6720\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-c2tg56ndlf7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n      \"name\": \"862cf1bb-7dd3-40c1-a414-17b3b0d583bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:25.770566501Z\",\n      \"endTime\": \"2026-10-20T05:15:25.516223632Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-storage-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, catalog-operator, cluster-version-operator, ingress-operator, olm-operator, openshift-route-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/862cf1bb-7dd3-40c1-a414-17b3b0d583bd\n            correlation ID: d433d7b9-c808-4842-80b1-be90eb0e3f4a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"4a8ff495293973ab2315985d325a1ead08660f0d23a49dee4c3f8735bfcc49eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6ecbe2fdf10383ad3e6b4b9603afc5bfad8be6ed12177f6a3782005a280d5e7a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ignition-server-proxy, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e05109c4140\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-7cbm7pwl25bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n      \"name\": \"c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:17:40.509470506Z\",\n      \"endTime\": \"2026-10-20T06:46:40.233866938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n            correlation ID: 3a5074fe-d20a-46db-b752-31f0ad8d374b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e05109c4140\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-7cbm7pwl25bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n      \"name\": \"c7aa03ac-1598-47ec-a13f-ec7f148e56dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:17:40.509470506Z\",\n      \"endTime\": \"2026-10-20T06:46:40.233866938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c7aa03ac-1598-47ec-a13f-ec7f148e56dd\n            correlation ID: 3a5074fe-d20a-46db-b752-31f0ad8d374b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"d3b3192ea30a5bd5e470657add4a5cc9c892b2e2b76b49698e1203d4b6060403","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8bfd122bfb355a24d92287df3269e0a7f0e6a9d4d45c80a03f8103457622a6b5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e9e304bc800\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hrm4cs7zffg2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n      \"name\": \"a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:15.624065092Z\",\n      \"endTime\": \"2026-10-20T05:15:15.524291644Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n            correlation ID: 8327c480-3421-4e24-a66c-3cea0477dfa6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e9e304bc800\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hrm4cs7zffg2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n      \"name\": \"a343792f-4e4d-47a0-a712-41d3ec54b93a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T04:46:15.624065092Z\",\n      \"endTime\": \"2026-10-20T05:15:15.524291644Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a343792f-4e4d-47a0-a712-41d3ec54b93a\n            correlation ID: 8327c480-3421-4e24-a66c-3cea0477dfa6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"6f9537cede3126ce1bf541078c4af085af4c75317ec382bbf40d8c920322279b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"5a8c81cfc5de57e2ac5cb1f344b5d4c790956481a2f66094e81c030894d26e74","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fffa36f85bbbff9b9e35ac04e5e1995d6605346e9b86d9872df10486fda0e14f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a7827d5ad5ffd2899a170b78f82132d6020ab7d4fd7644e60ac3a32d23a3e66e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"600ad974be9c7777150a3432faf4397d3733a0f1f567bf0b6ee9a4202295378e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28688821ac40\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-69bxttgtnjdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\",\n      \"name\": \"81fcc58f-c93e-4c14-8462-568e76b1f007\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:29:44.602488636Z\",\n      \"endTime\": \"2026-10-20T06:58:50.194412463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.w4p3g9f2y8m5o4t.2kot.eastus2euap.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n            correlation ID: 57aab543-83b3-413d-b1ad-731d8e533b5f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28688821ac40\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-69bxttgtnjdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\",\n      \"name\": \"81fcc58f-c93e-4c14-8462-568e76b1f007\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:29:44.602488636Z\",\n      \"endTime\": \"2026-10-20T06:58:50.194412463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.w4p3g9f2y8m5o4t.2kot.eastus2euap.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/81fcc58f-c93e-4c14-8462-568e76b1f007\n            correlation ID: 57aab543-83b3-413d-b1ad-731d8e533b5f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"f96c57f99ac73af4b4c3514a5ad2d2bc098868a52ebe93532ace21aa2781fffd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18083b0da87c5f910711dbd500417dc67679d796a27ea6259b1d8bd9fe4e29dd","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2072efeace0\u003e: \n    failed to create HCP cluster ea-listmlx87m: failed waiting for cluster=\"ea-listmlx87m\" in resourcegroup=\"ea-list-nlprzq99rzmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n      \"name\": \"9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:37:03.327418936Z\",\n      \"endTime\": \"2026-10-20T06:06:10.675381294Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 14m23s ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n            correlation ID: b09d3f5f-9c93-4d50-8097-63a061652137\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2072efeace0\u003e: \n    failed to create HCP cluster ea-listmlx87m: failed waiting for cluster=\"ea-listmlx87m\" in resourcegroup=\"ea-list-nlprzq99rzmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n      \"name\": \"9cb322ee-80e1-4c68-9a22-2c73b38798de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T05:37:03.327418936Z\",\n      \"endTime\": \"2026-10-20T06:06:10.675381294Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 14m23s ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9cb322ee-80e1-4c68-9a22-2c73b38798de\n            correlation ID: b09d3f5f-9c93-4d50-8097-63a061652137\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"f1d59441d9727804d5c781df8711ee40935778aa3e5e87010831b826d8329748","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"9d48f25db5125a951413a47642b0efecb6d5d06c908e8b1e5fde16b1b20e1440","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-shckcq with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17cd02074520\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-shckcq: failed waiting for cluster=\"np-version-upgrade-cluster-shckcq\" in resourcegroup=\"rg-np-version-upgrade-shckcq-xnqxw8phxcj5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n      \"name\": \"33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:25:29.730721891Z\",\n      \"endTime\": \"2026-10-20T06:54:30.236471474Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n            correlation ID: 5197a65a-354b-4a54-a076-091fbd77ba87\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-shckcq with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17cd02074520\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-shckcq: failed waiting for cluster=\"np-version-upgrade-cluster-shckcq\" in resourcegroup=\"rg-np-version-upgrade-shckcq-xnqxw8phxcj5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n      \"name\": \"33f5f50d-a3c9-4760-b56c-deee7ac82026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-20T06:25:29.730721891Z\",\n      \"endTime\": \"2026-10-20T06:54:30.236471474Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/33f5f50d-a3c9-4760-b56c-deee7ac82026\n            correlation ID: 5197a65a-354b-4a54-a076-091fbd77ba87\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"9572b1168dd4e09bd2006ea91b7e7cbda85a1354aa16390899d53973f05cccba","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-15T06:27:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x29388dc21b20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l6ddnj7vhmkq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"name\": \"8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T16:34:55.668662338Z\",\n      \"endTime\": \"2026-10-21T16:53:59.759177283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n            correlation ID: 7627942b-6e2a-4d1f-a2c9-7a7ff2104478\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x29388dc21b20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l6ddnj7vhmkq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"name\": \"8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T16:34:55.668662338Z\",\n      \"endTime\": \"2026-10-21T16:53:59.759177283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n            correlation ID: 7627942b-6e2a-4d1f-a2c9-7a7ff2104478\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112937268917309440","occurred_at":"2026-10-21T16:01:15Z","signature_id":"6f8fa1557de794cf4f88a42a92286c03dca64dacbbf713b8b027b7822d5e2b70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6fe02653aee74c57d72dfff05e4f572539da57d10c19d6d05ba4d0781d80c774","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n      \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n            correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n      \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n            correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113594207997267968","occurred_at":"2026-10-23T11:31:42Z","signature_id":"a4e598ab2e945986cd1ad69b1bb3c65fbdf94c05e97f34879cec8e6778f23b99","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b9ac8e13134319c4cdeffb9d46f43716baeb67c5239b74632e6bf7675546e7af","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"bace6221e58a5d21dfe413ecbb1909743fd07f6b8c34c992d650b4df4119cf4a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ef96fcb3cedbbf63865b36d0e47bdd70f7edc69989eb675136e8b13a4ecb570d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a837c45580\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m8kxkqwbf5w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"name\": \"64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T09:39:04.403310399Z\",\n      \"endTime\": \"2026-10-25T09:58:04.042996716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n            correlation ID: a329644d-c403-4d74-b2b5-fcc949c9882c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a837c45580\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m8kxkqwbf5w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"name\": \"64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T09:39:04.403310399Z\",\n      \"endTime\": \"2026-10-25T09:58:04.042996716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n            correlation ID: a329644d-c403-4d74-b2b5-fcc949c9882c\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"eb4c9f692ebf3a9ebdd322c6273f89d4ee6b231c71d079f6aa6d3f3c0db70247","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-17T23:00:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"ddcef7f1372fb43fa0a1e2ed4da6ae9b117c2451538d5508f48d3045f3f05809","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d915ac72cff72053dd10981d7bc43a67a03525e9c88720e973363870db7ed357","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'australiaeast' and API version '2026-10-01' for type 'HCPOpenShiftClusters'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, centraluseuap, eastus2euap'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1976922010a0\u003e: \n    failed to create HCP cluster des-encrypt: failed starting cluster creation \"des-encrypt\" in resourcegroup=\"des-encrypt-4n7f599ndw26\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-4n7f599ndw26/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/des-encrypt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'australiaeast' and API version '2026-10-01' for type 'HCPOpenShiftClusters'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, centraluseuap, eastus2euap'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-4n7f599ndw26/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/des-encrypt\n            correlation ID: 90fe650f-dd4d-4539-aefc-1901e9671e38\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1976922010a0\u003e: \n    failed to create HCP cluster des-encrypt: failed starting cluster creation \"des-encrypt\" in resourcegroup=\"des-encrypt-4n7f599ndw26\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-4n7f599ndw26/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/des-encrypt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'australiaeast' and API version '2026-10-01' for type 'HCPOpenShiftClusters'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, centraluseuap, eastus2euap'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-4n7f599ndw26/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/des-encrypt\n            correlation ID: 90fe650f-dd4d-4539-aefc-1901e9671e38\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112000853891092480","occurred_at":"2026-10-19T02:00:17Z","signature_id":"913f32f568b94f7a2128390c0eb07bb91dacea40d6caf14dfa235677bd9cc882","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-14T23:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"e854c2baa0b762928bd1753c83c278e1be7a0f0d62fdeaf4af4e9edab0cc8f65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ebb8d128ac36a20abe9223f2c5925135d7f381d45a13394b4f2869aab4d48252","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n            observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n            observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"79dfce6cb10e96a30452837453f16449a11349eb353010336138903c85b23a84","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"40fd7a6950bf8f4a3acf4f0624934387f01a457fbf420dfeb26442bd2e3882ba","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-18T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2064b1710780\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.009s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.009s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2064b1710700\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2064b17106a0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2064b1710780\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.009s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.009s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2064b1710700\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x2064b17106a0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112000853891092480","occurred_at":"2026-10-19T02:00:17Z","signature_id":"954dda84de09af750bb594f544dfb635f834fc5301115a2ca74ccfddfbef1c3e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b6694bcdf5bd78471f8a6f6a55664c9b2d5e3ffa6e0b99c19a0b7d948e44b177","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb243792ee20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xb243792edc0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb243792ee20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xb243792edc0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113042690441482240","occurred_at":"2026-10-21T23:00:10Z","signature_id":"c7f43354740e69178951a6c68e22539a605df8b0aa7742a05b7005bd1faa97a1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6de12f7229509a8873177b6aee486cac0c911989cb81031037d884ebc7dad3dd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3866dcc37c20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x3866dc67cd90\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3866dcc37c20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x3866dc67cd90\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113033513308524544","occurred_at":"2026-10-21T22:23:42Z","signature_id":"3aee454bda0345aa80151b07bb5e4c51af83dae06c70040ad111bd3819e733e7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ec625aa187990188920942c47358d3eb1c7eda37bdf66a6d43c4b3578a43526","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x2d202c117968\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x2d202c117968\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"9dfcee82299c274c6cbfd06b3f9fb3c7b36ad00e636501927c19be2e530212db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d94cc015dc95597b28dc0d7b9906b72de1a3ac50e629b544d55f9dc2f683d21d","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bmk4fl56snsp/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112399316868927488","occurred_at":"2026-10-20T04:23:38Z","signature_id":"7626336d8319bd1b9484bd3d83ed4edd851ca8da911ba5db554a4e3dc064cc49","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5a972289ad4ab45b4713665dee826b535d8a68c11c1b3e9a148a15bbd67840c0","failure_pattern":"external auth \u003cexternal-auth\u003e.Status was nil","search_query":"external auth Properties.Status was nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-18T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.003s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.003s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112000853891092480","occurred_at":"2026-10-19T02:00:17Z","signature_id":"b0d3841d4a984de3c5ebae440c2e284dfd5fa10e426e984646f0504573da4d44","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"12318b0632fb674fc784fdd096e34ba234440b0cb9593f4ff4c36061f31941fe","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" in cluster=\"\u003ccluster\u003e\" resourcegroup=\"\u003cresource-group\u003e\" to finish updating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.694915254237288,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-18..2026-10-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6b805215ae5be9a20646349cb83be2fc300cf869ef7d92e96198f6d3b0f5e8fe","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}