{"meta":{"date":"2026-10-22","timezone":"UTC","generated_at":"2026-10-28T23:55:45Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":11,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":5,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113413922286473216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T23:35:19Z","started_at":"2026-10-22T23:35:19Z","completed_at":"2026-10-23T00:05:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:22:26.297077903Z","timing_metadata_checked_at":"2026-10-23T00:22:26.297077903Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T00:22:26.30065259Z","region_metadata_checked_at":"2026-10-23T01:24:51.126822267Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b05bf41057d0947814334846ff4b0806aec20ec3dd934b5dcb21bf0ee23a9dee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113413922286473216","occurred_at":"2026-10-22T23:35:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T23:00:29Z","started_at":"2026-10-22T23:00:29Z","completed_at":"2026-10-23T00:47:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:53:36.172121877Z","timing_metadata_checked_at":"2026-10-23T00:53:36.172121877Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:53:36.17548907Z","region_metadata_checked_at":"2026-10-23T00:53:36.17548907Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0413bd6f3a1e5afa9c0c80d77e4dce54422383293ad3e0d14084889dd9640752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"d6194b621062ac0dc7417d99a6662052b73f0727482224784de827ab3d686809","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8dce7251b0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8dce7251b0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"15466f414b90eaf42bf80faae50824bcf914c6a6b7308111b4d630690ef10240","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"554520898e3bf65aa33d08c67763b5d08098c2dea88239fbc79acfa28b30250c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe4d0c558c40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe4d0c558c40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"1ab15151456907c2b2f2ed8d367d53609112c07eae215ad76ede9325ec1ed987","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"a0af21048f0851309014ba5e3dc0d0f72ace62a5a58aa35d7e2b48b85ae30c13","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-5j4sbg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26e6691a0320\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26e6691a0300\u003e{\n            observation: \u003c*errors.errorString | 0x26e66a200e10\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26e6691a02c0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-5j4sbg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26e6691a0320\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26e6691a0300\u003e{\n            observation: \u003c*errors.errorString | 0x26e66a200e10\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26e6691a02c0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"2e991fb50ef4394342a5fd1cf4f46b24bb76d1605ae032d5cec2d75991ceb69c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"7d7a4bc924021f4fa04d801c41a3ac27180abe1576c70ed76bd677cebe7f2a09","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-w29x8s\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2369de188ea0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2369de188e60\u003e{\n            observation: \u003c*errors.errorString | 0x2369e043f830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2369de188d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-w29x8s\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2369de188ea0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2369de188e60\u003e{\n            observation: \u003c*errors.errorString | 0x2369e043f830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2369de188d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"36ff4cefb8784a72f1f17e6997f3a7401610b435e040ea0502cd4c6f167dcf34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"ad2974ff484501a74259c346bde4f86a21cbaef053d5d94585993a7186adc875","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x663da188400\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x663da188400\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"73d2606a8fc9f97ef3115545af3cdc53ab7913610c2ca52c2a79d6eedd92413a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"28f82d6598d8560878485b881cedade2766f6f99a7665f52a5642ec9863e8f99","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x14719e7fa150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x14719e7fa150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"b3b4add69f9d62162ddd129ac22ffacacbb8901238c493362b028c7dac73ce59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"11a1ea4698faced47725bf493b4aacc2124922e05d2aed8e74789e91de7a5717","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-zn6lkv\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1288c746e940\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x1288c746e900\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x1288c746e8c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x1288c8648ef0\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-zn6lkv\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1288c746e940\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x1288c746e900\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x1288c746e8c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x1288c8648ef0\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"ba6da9994956f63e83bae865ece4210c3c00bd995eb2904ece3a610bbb493dec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"91144e10f499ef01ce91bc27b79924e73fc51fedcf3381a2242dc3972585a568","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x37eb40681150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x37eb40681150\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"bec677b12cd05c27ddfdfef4c086cdd4d42f2d30b31ded24d9dcbca9e2d409f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"e51db32a50ceffa0ea6a335e06c32c375c606cf3ce2f73a2c6b002fa3305aea2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-20-z2m2qz\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5840fb59bc0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x5840fb59b40\u003e{\n            observation: \u003c*errors.errorString | 0x584102eb3b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x5840fb59ac0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-20-z2m2qz\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5840fb59bc0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x5840fb59b40\u003e{\n            observation: \u003c*errors.errorString | 0x584102eb3b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x5840fb59ac0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"e1e0014dac086384e4344250f1438a09d641d8219fdd854558e0cd47b75ab3cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","failed_at":"e2e","signature_id":"60d8736178a39e3a5b4fef671fa2ef1330288eda4204ffbc60e31862e2471d95","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9frg2v\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26bc41040e40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26bc41040dc0\u003e{\n            observation: \u003c*errors.errorString | 0x26bc40ab97d0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26bc41040d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9frg2v\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x26bc41040e40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x26bc41040dc0\u003e{\n            observation: \u003c*errors.errorString | 0x26bc40ab97d0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x26bc41040d80\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113346401319522304","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T19:06:58Z","started_at":"2026-10-22T19:06:58Z","completed_at":"2026-10-22T19:37:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:48:51.074937151Z","timing_metadata_checked_at":"2026-10-22T19:48:51.074937151Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-22T19:48:51.077905326Z","region_metadata_checked_at":"2026-10-22T20:50:44.878820847Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0c3abab17532c8ad6cb71276264b1e2a6c55b846f62a03d7cba0f86bbd538c3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113346401319522304","occurred_at":"2026-10-22T19:06:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T14:20:40Z","started_at":"2026-10-22T14:20:40Z","completed_at":"2026-10-22T20:15:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:30:21.427985897Z","timing_metadata_checked_at":"2026-10-22T20:30:21.427985897Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:30:21.431483687Z","region_metadata_checked_at":"2026-10-22T20:30:21.431483687Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"2662fdb21d321c85162e970197a0635649b3454a1ae1625a7ef61b11fb95e694","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"dd0b0b8df7b2166bc054ede8f1d58b688dc0dac527d6b2fd66ecb6ab9cb8ddc7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15ce09908640\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30507e2e-a247-48d9-a1d3-5e74fda3481f\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30507e2e-a247-48d9-a1d3-5e74fda3481f\n            correlation ID: bd254d01-316e-4e34-af7b-a66cb2bc888c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15ce09908640\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30507e2e-a247-48d9-a1d3-5e74fda3481f\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30507e2e-a247-48d9-a1d3-5e74fda3481f\n            correlation ID: bd254d01-316e-4e34-af7b-a66cb2bc888c\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"4b3b8cb94eba5eed790e36dfb32f4ef88cfcee3298c6b434da15c6b2885cbc52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"4fa8b7fc464071338bdd4561210059a3e2c75c44318bf3c18a41e27d54116b38","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"56b65da71ab5d0df8c6e632d325d83241f38b785e98822e9f0ed267d5fa50f6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"6b805215ae5be9a20646349cb83be2fc300cf869ef7d92e96198f6d3b0f5e8fe","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"12318b0632fb674fc784fdd096e34ba234440b0cb9593f4ff4c36061f31941fe","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" in cluster=\"\u003ccluster\u003e\" resourcegroup=\"\u003cresource-group\u003e\" to finish updating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b"}},{"row_id":"754726aaed70439c99781f4589f4bfe658e50b18baa52a408f6a1eaaa9043889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"2cc2591b74b05456e73089bc9b36c0f1f7eee60669fc135f3ce15bfec52d2937","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"8f1862c84eb6feb03d3f64c5298f6fea19866828187c8048c6b4c4df987003b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"b8acb5f0bafdb5a3735bb27828a6b4f3c5003359238914449eaa893ab955598b","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6fc68b1db40\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-xkft46\" in resourcegroup=\"rg-np-downgrade-xkft46-4q28hv2srddf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fb2f9bf6-57ed-44ee-a29f-f7d739da032d\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fb2f9bf6-57ed-44ee-a29f-f7d739da032d\n            correlation ID: f554d9ad-22a3-4412-a6b0-c83016dc310e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6fc68b1db40\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-xkft46\" in resourcegroup=\"rg-np-downgrade-xkft46-4q28hv2srddf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fb2f9bf6-57ed-44ee-a29f-f7d739da032d\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fb2f9bf6-57ed-44ee-a29f-f7d739da032d\n            correlation ID: f554d9ad-22a3-4412-a6b0-c83016dc310e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"9630a05a316e5b7df3a1db89b40270605374fd5a351c23b991fccecef1cdab45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"fe77c534f2d11553fbb56bdbdd25fa10e6b63e20abc56954df520c44f8dde521","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x32d7aaca6180\u003e: \n    resource group \"nodepool-delete-cd2q42rnhv9p\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cd2q42rnhv9p\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8abc07fd-a04c-472f-9209-d29535531a3b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8abc07fd-a04c-472f-9209-d29535531a3b\n            correlation ID: 270a5375-3ee5-4792-82aa-8de0a5f92543\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x32d7aaca6180\u003e: \n    resource group \"nodepool-delete-cd2q42rnhv9p\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cd2q42rnhv9p\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8abc07fd-a04c-472f-9209-d29535531a3b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8abc07fd-a04c-472f-9209-d29535531a3b\n            correlation ID: 270a5375-3ee5-4792-82aa-8de0a5f92543\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"b79c52a83df7fd3ca0265c65cb63e8d554f0ff890b9d86587dae67cfc7b646d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"6f30c7a8231594aeb46f3d54216c8a117b477376ff72a9a27924ceba4f803e6f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-rt5grp\nUnexpected error:\n    \u003c*fmt.wrapError | 0x248e5dd5cbc0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-rt5grp\" in resourcegroup=\"rg-np-version-upgrade-rt5grp-4q86fvkpwgjw\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5874d223-8094-4a83-b531-392c43d65224\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5874d223-8094-4a83-b531-392c43d65224\n            correlation ID: 2733429a-1831-4753-b8fd-9cd662863c39\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-rt5grp\nUnexpected error:\n    \u003c*fmt.wrapError | 0x248e5dd5cbc0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-rt5grp\" in resourcegroup=\"rg-np-version-upgrade-rt5grp-4q86fvkpwgjw\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5874d223-8094-4a83-b531-392c43d65224\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5874d223-8094-4a83-b531-392c43d65224\n            correlation ID: 2733429a-1831-4753-b8fd-9cd662863c39\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"b8282638886f33b16257667b83c1286613ee4b968ac2052deb4f2df104fa16be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"3fb0e1297078af590b2d19465e2eb4a0832651168d99f0176c33e483a935dcbf","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"d00119f82f26dddc56244395fded5a69ebe268257cd8633fa74988b395a1038c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"fd25ada7f7c17331da353bb415e7abc6bfe24733b76e6dae5911dba3aa007638","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x42a73e640c0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-cqs6bm\" in resourcegroup=\"rg-np-version-upgrade-cqs6bm-bdk46hm6gz2t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7be4883-49a5-4223-a7fd-b85c36625889\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7be4883-49a5-4223-a7fd-b85c36625889\n            correlation ID: e1f797c4-9d76-4caf-85e3-d444fb84110b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x42a73e640c0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-cqs6bm\" in resourcegroup=\"rg-np-version-upgrade-cqs6bm-bdk46hm6gz2t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7be4883-49a5-4223-a7fd-b85c36625889\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7be4883-49a5-4223-a7fd-b85c36625889\n            correlation ID: e1f797c4-9d76-4caf-85e3-d444fb84110b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"d2bea39e7a599e13466eb150d429ea95b3947614f949fd5ec2ecf3651dce82a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"6d820660123e1f2b46bada2acc37a3d7025031ef13070c0759df5cdf0c7b2de4","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:143]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1628bc606300\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342\n            correlation ID: 6c7f6da2-0648-48d3-a365-326009a54a6d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:143]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1628bc606300\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342\n            correlation ID: 6c7f6da2-0648-48d3-a365-326009a54a6d\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"d5b40fe79d6a143a3ef56d65b01d35c3b94bdaa107eec63558ba3aafe908ecdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"0705e7ce0972714a7b11f80b2c8aff85f61cdc6febe2637f5a146fe039b1f537","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"df8f8b5f3897f3922ced5d84e82b36d97f197b0c858ce179fc7a7e115a3c3f74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"06ff6d227a4a567e0691243be5eccb0caeeaaa315bcc42dc0fea399f15c10736","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}},{"row_id":"e7c7ba9e587234fe6ef9283bee902b8ba6d3cd5db418ac54cca20d7db3d8b832","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"022548c08c7f3d00dcc2c6ccfbb90c7621484b0f6cc92b83282d239b2bf97fec","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"f00975b50f145d06048c20708c2f9192dd2cfedbce4f207daee5a250ce241212","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"e4a1953e3872ab73515a0438ce007de9e2def00ae922c3087992e0fce1014e4c","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23e7733bb0a0\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-q27q4m\" in resourcegroup=\"rg-np-dg-minor-q27q4m-lfz4ljlvt6cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c2dbec-e136-4d1f-8130-7b66c5f0a6c1\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c2dbec-e136-4d1f-8130-7b66c5f0a6c1\n            correlation ID: 4b13a3a4-dc33-47f4-9b01-e6df32158296\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23e7733bb0a0\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-q27q4m\" in resourcegroup=\"rg-np-dg-minor-q27q4m-lfz4ljlvt6cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c2dbec-e136-4d1f-8130-7b66c5f0a6c1\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c2dbec-e136-4d1f-8130-7b66c5f0a6c1\n            correlation ID: 4b13a3a4-dc33-47f4-9b01-e6df32158296\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342"}},{"row_id":"f3557580673ffd575c73d849cf9b065c952ab6826aa89ba03252f5f99149cf8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","failed_at":"e2e","signature_id":"aca4445f1d5148a20ae5983cba5de62c36b6c0a7b69db5f707a5e755bb5182a6","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x288e5c900060\u003e: \n    resource group \"ephemeral-osdisk-4gswgswxhhgn\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4gswgswxhhgn\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e9d42f84-ef34-4bd9-97b4-3ce13c3377ef\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e9d42f84-ef34-4bd9-97b4-3ce13c3377ef\n            correlation ID: c098aa4e-9c5f-48a5-ac6b-501930ee0285\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x288e5c900060\u003e: \n    resource group \"ephemeral-osdisk-4gswgswxhhgn\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4gswgswxhhgn\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e9d42f84-ef34-4bd9-97b4-3ce13c3377ef\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e9d42f84-ef34-4bd9-97b4-3ce13c3377ef\n            correlation ID: c098aa4e-9c5f-48a5-ac6b-501930ee0285\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["12318b0632fb674fc784fdd096e34ba234440b0cb9593f4ff4c36061f31941fe","3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113179004134494208","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T08:01:40Z","started_at":"2026-10-22T08:01:40Z","completed_at":"2026-10-22T10:31:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T10:46:11.54617072Z","timing_metadata_checked_at":"2026-10-22T10:46:11.54617072Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T10:46:11.55018991Z","region_metadata_checked_at":"2026-10-22T10:46:11.55018991Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T04:22:40Z","started_at":"2026-10-22T04:22:40Z","completed_at":"2026-10-22T06:52:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:53:17.755561619Z","timing_metadata_checked_at":"2026-10-22T06:53:17.755561619Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:53:17.758324013Z","region_metadata_checked_at":"2026-10-22T06:53:17.758324013Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1e3ba50a18d645f28d1e5ec4a40a9b1eaddc1d9421c8211ab79324335f2163c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","failed_at":"e2e","signature_id":"bace6221e58a5d21dfe413ecbb1909743fd07f6b8c34c992d650b4df4119cf4a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b9ac8e13134319c4cdeffb9d46f43716baeb67c5239b74632e6bf7675546e7af","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"52e28d1626e3422346649d5d6c9e590b35be1520d1eafed91323c418dcd6d16f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","failed_at":"e2e","signature_id":"8a9ded4e992ead494e672f2c33210492690aab0b14e2de539e32782deff72275","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","b9ac8e13134319c4cdeffb9d46f43716baeb67c5239b74632e6bf7675546e7af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123585936592896","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T04:21:21Z","started_at":"2026-10-22T04:21:21Z","completed_at":"2026-10-22T06:37:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:53:17.564799732Z","timing_metadata_checked_at":"2026-10-22T06:53:17.564799732Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:53:17.567608426Z","region_metadata_checked_at":"2026-10-22T06:53:17.567608426Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113122909672181760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T04:18:56Z","started_at":"2026-10-22T04:18:56Z","completed_at":"2026-10-22T06:35:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:53:17.340429719Z","timing_metadata_checked_at":"2026-10-22T06:53:17.340429719Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:53:17.343400012Z","region_metadata_checked_at":"2026-10-22T06:53:17.343400012Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113087989226672128","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T02:00:10Z","started_at":"2026-10-22T02:00:10Z","completed_at":"2026-10-22T03:54:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T04:11:48.155707427Z","timing_metadata_checked_at":"2026-10-22T04:11:48.155707427Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T04:11:48.158979211Z","region_metadata_checked_at":"2026-10-22T04:11:48.158979211Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113078612994297856","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T01:22:55Z","started_at":"2026-10-22T01:22:55Z","completed_at":"2026-10-22T03:23:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T03:40:59.659810969Z","timing_metadata_checked_at":"2026-10-22T03:40:59.659810969Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T03:40:59.664532355Z","region_metadata_checked_at":"2026-10-22T03:40:59.664532355Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T01:09:23Z","started_at":"2026-10-22T01:09:23Z","completed_at":"2026-10-22T04:17:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T04:32:48.427994789Z","timing_metadata_checked_at":"2026-10-22T04:32:48.427994789Z","region":"eastus2euap","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T04:32:48.431755079Z","region_metadata_checked_at":"2026-10-22T04:32:48.431755079Z"},"failed_at":["e2e"],"failed_test_count":62,"occurrences":[{"row_id":"0227c5dcba470aafdd1afc7377067e31af835e21701be9b8d620ed5145a4d2bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"f7d5f9aaa39bc4f25aca130a942f0670850a3d28be7139dc0a9df1e58b8e9577","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ecb44a0c500\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-s6r6htggh6z7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\",\n      \"name\": \"2fc31b58-fcb3-4fef-814b-d7ab54f186d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:05.760617049Z\",\n      \"endTime\": \"2026-10-22T02:03:06.678299552Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\n            correlation ID: bb4615a3-e6b5-4e4b-a502-f87305616869\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ecb44a0c500\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-s6r6htggh6z7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\",\n      \"name\": \"2fc31b58-fcb3-4fef-814b-d7ab54f186d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:05.760617049Z\",\n      \"endTime\": \"2026-10-22T02:03:06.678299552Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2fc31b58-fcb3-4fef-814b-d7ab54f186d7\n            correlation ID: bb4615a3-e6b5-4e4b-a502-f87305616869\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"05d84d39fc4a2994f5a32797d3b1deea1450872e7985b248a597032056efb784","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"d2d766d5aec5f079fb45688d592a83dedffde25333099326b9f7f5f8e100d8f2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-bzlw46\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202d14170440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bzlw46: failed waiting for cluster=\"cluster-zstream-5-0-bzlw46\" in resourcegroup=\"rg-zstream-upgrade-5-0-l5ckf5pqgjzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"name\": \"b119ec20-94fb-4593-816b-4d56620a6799\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:58.803577295Z\",\n      \"endTime\": \"2026-10-22T02:52:00.551152403Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-storage-operator, redhat-operators-catalog, capi-provider, cluster-network-operator, ingress-operator, olm-collect-profiles, openshift-controller-manager, azure-cloud-controller-manager, kube-storage-version-migrator, cluster-image-registry-operator, konnectivity-agent, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, catalog-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-policy-controller, olm-operator, packageserver, community-operators-catalog, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b119ec20-94fb-4593-816b-4d56620a6799\n            correlation ID: cb65955f-e919-441d-963c-27a6a9495a9b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0fda012c0a6d398610872ebbb572fd6956cb338d299f2136cf4e267aeafbf6fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"541341a5ff6a403f156c133d1168366027c4c77d725975906770d24cc82a1c34","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-mszzh7\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11efeacc6c00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-mszzh7: failed waiting for cluster=\"cp-ystream-upgrade-5-1-mszzh7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-w6jlwhdqxzx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\",\n      \"name\": \"6e60be32-8991-4fef-8fa2-b73c032aca4f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:24:14.277639419Z\",\n      \"endTime\": \"2026-10-22T02:53:20.714905396Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\n            correlation ID: 58eae255-dc55-44cf-aba8-2164cd64b6d5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-mszzh7\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11efeacc6c00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-mszzh7: failed waiting for cluster=\"cp-ystream-upgrade-5-1-mszzh7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-w6jlwhdqxzx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\",\n      \"name\": \"6e60be32-8991-4fef-8fa2-b73c032aca4f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:24:14.277639419Z\",\n      \"endTime\": \"2026-10-22T02:53:20.714905396Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e60be32-8991-4fef-8fa2-b73c032aca4f\n            correlation ID: 58eae255-dc55-44cf-aba8-2164cd64b6d5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"10f179237402aeddc1fbb2f8366a6c4b3137f5714fb76fe16e14fb599f067436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"2ca8d85c3dc2b47b78c6a24945b274f76b13627c62c8a5c01d3ebc927fad0021","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbec85184960\u003e: \n    failed to create HCP cluster ea-listr985hw: failed waiting for cluster=\"ea-listr985hw\" in resourcegroup=\"ea-list-psnhlv4lhlmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\",\n      \"name\": \"d538c387-3fea-488a-a164-ab083059af27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:04.906647307Z\",\n      \"endTime\": \"2026-10-22T02:54:10.540852911Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d538c387-3fea-488a-a164-ab083059af27\n            correlation ID: 924be768-33e9-47c5-9219-bd9273d15ea1\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1533693045ee113a3977d8ebf3f3a25584d04f658c130a1c88e53612c063b980","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"30b3256251eebbc6ad2f2d21745e8e3e6c242862e551efcc5662d3dab722db4f","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a75f4e12f60\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-fx25mzlnzj9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"name\": \"0f9269de-192b-41ce-bc41-692ff3dda38f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:16.144784385Z\",\n      \"endTime\": \"2026-10-22T02:03:16.753815438Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f9269de-192b-41ce-bc41-692ff3dda38f\n            correlation ID: 0c97842f-0279-4b59-9196-c2a2e8ab56cb\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"15ea72eb7bb3f3ff48789ef952ffa94aceaf4087e1f525c7b62fe47d9f13a873","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"884359bc37b7b0f624f652c6e3ab6489f45271d6a5f2eaf45cf5d681cd51b93c","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c787f4ae5a0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-s42wrnh68zb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\",\n      \"name\": \"5da49df1-032c-48e7-8170-21123b884de0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:02:14.362284979Z\",\n      \"endTime\": \"2026-10-22T03:31:15.143035388Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, konnectivity-agent, packageserver, control-plane-pki-operator, azure-cloud-controller-manager, machine-approver, cluster-version-operator, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller, cluster-image-registry-operator, catalog-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\n            correlation ID: 8ab168b3-b3fc-4668-8ddc-5c5d1b6b46a0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c787f4ae5a0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-s42wrnh68zb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\",\n      \"name\": \"5da49df1-032c-48e7-8170-21123b884de0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:02:14.362284979Z\",\n      \"endTime\": \"2026-10-22T03:31:15.143035388Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, konnectivity-agent, packageserver, control-plane-pki-operator, azure-cloud-controller-manager, machine-approver, cluster-version-operator, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-policy-controller, cluster-image-registry-operator, catalog-operator, community-operators-catalog, certified-operators-catalog, cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5da49df1-032c-48e7-8170-21123b884de0\n            correlation ID: 8ab168b3-b3fc-4668-8ddc-5c5d1b6b46a0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"18ea29221cd08651d65f69cf6d93ae0544f66f56a7a44f7645db7bbc59ef5b61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"8443649e9083d1ab2db3cf3952a87eb977678118008a762dcd368311d2aaac93","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x383d8a0a0760\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4hpbwr6m2mc9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\",\n      \"name\": \"fb9bf18a-3669-464c-b675-17049049d397\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:04.492681455Z\",\n      \"endTime\": \"2026-10-22T02:03:06.687321625Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\n            correlation ID: fd8633cb-5d10-4988-961a-d49a1e752197\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x383d8a0a0760\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4hpbwr6m2mc9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\",\n      \"name\": \"fb9bf18a-3669-464c-b675-17049049d397\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:04.492681455Z\",\n      \"endTime\": \"2026-10-22T02:03:06.687321625Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fb9bf18a-3669-464c-b675-17049049d397\n            correlation ID: fd8633cb-5d10-4988-961a-d49a1e752197\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1db82fa4fdb9cf3d2a62bf0079cc2821db5bd9668dae06f6793c6608e781aa4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"5a8c81cfc5de57e2ac5cb1f344b5d4c790956481a2f66094e81c030894d26e74","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"279ee4114efc25c3576865416e0b3e9a9e1e1bba3d44f5f84ce0af1b1d344674","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"3869c78574390577fcc299556aef741fbec4d0cca2bc649c0c2a2e1cd5f36f04","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:127]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x119df757e3e0\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-hzm47sjhc426\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"name\": \"6a6c258e-71e6-4920-8857-17ab7dc1e45f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:48.207242699Z\",\n      \"endTime\": \"2026-10-22T03:38:55.0934354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, dns-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, certified-operators-catalog, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, kube-controller-manager, kube-scheduler, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-policy-controller, packageserver, redhat-operators-catalog, control-plane-pki-operator, ingress-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a6c258e-71e6-4920-8857-17ab7dc1e45f\n            correlation ID: 80334f53-9a3b-4203-a7f4-9d0844089824\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2efa8c31ebb6970a6335eabc61a002c13d7d713c340e8d4c09542dfafae83e22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"fa2dafb46bcd964c71b90a558a536dd7120ddbfa5212144072e39dc9ca2ed0c5","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9e2f3a4840\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\",\n      \"name\": \"579a5520-85b0-4d76-8f4f-8f11c4f8426f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:51.321199008Z\",\n      \"endTime\": \"2026-10-22T02:03:56.79400182Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, azure-cloud-controller-manager, catalog-operator, redhat-marketplace-catalog, certified-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, cluster-version-operator, openshift-route-controller-manager, machine-approver, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\n            correlation ID: 02b44ad8-cedb-41d0-85b1-5d72c2719877\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9e2f3a4840\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\",\n      \"name\": \"579a5520-85b0-4d76-8f4f-8f11c4f8426f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:51.321199008Z\",\n      \"endTime\": \"2026-10-22T02:03:56.79400182Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, azure-cloud-controller-manager, catalog-operator, redhat-marketplace-catalog, certified-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, cluster-version-operator, openshift-route-controller-manager, machine-approver, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/579a5520-85b0-4d76-8f4f-8f11c4f8426f\n            correlation ID: 02b44ad8-cedb-41d0-85b1-5d72c2719877\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2f67b6f5f57ce1f0a70d598b0a2dcaa79dcf4822928c3bf0afa86465af1da1f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"e4e0808ae7571f998df5b39264b674c05cb041138f2cb2532613af052179af56","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaf2b1adc460\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-clfhrsrccp5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\",\n      \"name\": \"2f16d2a2-0de5-4840-9b52-f254619dcdd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:00.294362099Z\",\n      \"endTime\": \"2026-10-22T02:56:00.372773919Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\n            correlation ID: 8e6afc06-e48b-4d7e-a53d-fbab147992fd\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaf2b1adc460\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-clfhrsrccp5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\",\n      \"name\": \"2f16d2a2-0de5-4840-9b52-f254619dcdd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:00.294362099Z\",\n      \"endTime\": \"2026-10-22T02:56:00.372773919Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2f16d2a2-0de5-4840-9b52-f254619dcdd9\n            correlation ID: 8e6afc06-e48b-4d7e-a53d-fbab147992fd\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"31ea05c082866fd5c4ed85ddbf121835742b7a4d7089b1ee76aa862a1fe43fc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"a7827d5ad5ffd2899a170b78f82132d6020ab7d4fd7644e60ac3a32d23a3e66e","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fffa36f85bbbff9b9e35ac04e5e1995d6605346e9b86d9872df10486fda0e14f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3522ca89a5538d7a2a033283264f02f0df979178d27f43618cd307d0eb54e27e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"3b522484ec379b7598bbd268e9f04bacd0edc777033a068970a7c75bb7fd3ebc","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33ca21720880\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6zgdp7hhpk7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"name\": \"c707ea6e-b74b-443a-9b32-fe7f80a738f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:00.37199898Z\",\n      \"endTime\": \"2026-10-22T02:54:00.386898896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c707ea6e-b74b-443a-9b32-fe7f80a738f2\n            correlation ID: cd0ac0e6-4a75-4391-a2be-e65722141c43\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3a73985d3690796f6e5883945c93417922e42c8ffddf3010df694f3a8fac13a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"96c75f1ee40922482fa619e484710e1e93eb36cf6eaf5f78027744f7acdd09b8","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-dsklst\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19639643c920\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dsklst: failed waiting for cluster=\"cluster-zstream-4-22-dsklst\" in resourcegroup=\"rg-zstream-upgrade-4-22-5t2tjvkj99ps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"name\": \"13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:46.501521761Z\",\n      \"endTime\": \"2026-10-22T02:51:50.360464178Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-image-registry-operator, community-operators-catalog, machine-approver, azure-cloud-controller-manager, dns-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, konnectivity-agent, certified-operators-catalog, kube-storage-version-migrator, catalog-operator, capi-provider, openshift-apiserver, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, cluster-policy-controller, kube-scheduler, olm-collect-profiles, redhat-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f8c2f6-a8c6-4ec9-bef8-08b6e4a75a38\n            correlation ID: 4f330ac9-7640-45b9-b580-8c05eae65daf\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"455bb9d76e1b5453daec80f5517458eee5ab70f11de3654e1666a3cff8406590","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"58f7b8768cce6693ee21bd067a2fa340654f0f3236bb851bb76e41548e9f1d61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"b311e9b9b11e8b11ec8d0f86f4d29406e7d83165875e0863fb9281515f0c7523","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19104d7cc9c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-l8cntnddrw7h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\",\n      \"name\": \"990ea455-66b6-4073-a5f2-4aeddb743877\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:36:21.537163403Z\",\n      \"endTime\": \"2026-10-22T02:05:26.659061975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\n            correlation ID: 3c80a876-0ab2-4b84-be94-2ce70586894c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19104d7cc9c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-l8cntnddrw7h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\",\n      \"name\": \"990ea455-66b6-4073-a5f2-4aeddb743877\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:36:21.537163403Z\",\n      \"endTime\": \"2026-10-22T02:05:26.659061975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/990ea455-66b6-4073-a5f2-4aeddb743877\n            correlation ID: 3c80a876-0ab2-4b84-be94-2ce70586894c\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5b1d67cbaf0c57e3dd3a5a56b86f078db552d10abbc36b6e675804bf791c7760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"0e6ceee3cd3e93558b27b65ed689870c63b26dd3bc15ef1fe3541ac45579879e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-pp2v6j\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9f443c1ebc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-pp2v6j: failed waiting for cluster=\"cp-ystream-upgrade-4-22-pp2v6j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4brmx258wcvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"name\": \"c465a539-e0e2-4a15-96b0-b991fee60416\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:40.34684753Z\",\n      \"endTime\": \"2026-10-22T02:52:40.362346793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-storage-operator, cluster-policy-controller, packageserver, hosted-cluster-config-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, community-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, kube-scheduler, cluster-network-operator, dns-operator, konnectivity-agent, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c465a539-e0e2-4a15-96b0-b991fee60416\n            correlation ID: 81f12348-0a31-441f-af5f-8f18eef28715\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5c50df1a2055c3bca18e68289a0b61095bdfa96e04e3fae144d64f6d6b6bc61c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"51ee050adec069064368b692e2e9f92789ea23eb4ea27242862d06944731e6a7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x363c02e8080\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jbrqpz7ctxtq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"name\": \"21e6bfc0-fe4d-4dac-b056-c840f710f982\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:29.420591363Z\",\n      \"endTime\": \"2026-10-22T03:40:35.058626079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/21e6bfc0-fe4d-4dac-b056-c840f710f982\n            correlation ID: c21da3fc-8f7d-45b1-bbc4-5d3a6ee72767\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5e2476401ecd108212881f83321201a709661b2f96b2a6854e9e2154eff99136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"179000d9fd7136a36d03dbd3bfaa3c8edbf92638e8ad95333390e6c1eadcd7dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-2s5zlgwb6bct/cluster-candidate-5-1-84zdqd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16cd8e8f9880\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-84zdqd: failed waiting for cluster=\"cluster-candidate-5-1-84zdqd\" in resourcegroup=\"rg-candidate-5-1-2s5zlgwb6bct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"name\": \"9d897978-69ec-403d-ad9b-fd32c33747d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:55.892991844Z\",\n      \"endTime\": \"2026-10-22T02:51:00.378883542Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: machine-approver, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, kube-scheduler, dns-operator, konnectivity-agent, kube-controller-manager, cluster-storage-operator, cluster-autoscaler, cluster-policy-controller, hosted-cluster-config-operator, olm-operator, olm-collect-profiles, community-operators-catalog, cluster-version-operator, kube-storage-version-migrator, packageserver, control-plane-pki-operator, openshift-controller-manager, azure-cloud-controller-manager, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d897978-69ec-403d-ad9b-fd32c33747d4\n            correlation ID: bdf3de8a-4b44-40d3-b952-4e50b1bc65e3\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"654ce7e92166f30d89b71ada67f5d785738f3ae578eb4cb8f954bfdcd33a90d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"01479cc8260099169681c3c9aac7e1601c47f3919a6c41c57610a9af6d7dfa91","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-fvrddf at version 4.22.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b64a153a380\u003e: \n    failed to create HCP cluster np-skip-minor-fvrddf: failed waiting for cluster=\"np-skip-minor-fvrddf\" in resourcegroup=\"rg-np-skip-minor-fvrddf-fpw64csf8rm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"name\": \"fcd2c383-b881-499c-8bed-cb24dce8c319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:54.281108118Z\",\n      \"endTime\": \"2026-10-22T03:39:55.15624633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fcd2c383-b881-499c-8bed-cb24dce8c319\n            correlation ID: e6ae3233-540c-4141-81e8-0b055bb45703\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c1a1580fdd13d3ae1161524c2f671808de4047a5821da255226373943bb414d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"1e540a6273c70580b8d0a813a5ac2b3184c764a6be285464f22624d92e6da8b4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d4d33b70620\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-b5r6x68bzp5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"name\": \"a651d305-0bc4-4e50-8659-237a66cb9f85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:20:28.664519088Z\",\n      \"endTime\": \"2026-10-22T02:49:30.367118005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, control-plane-pki-operator, catalog-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-node-tuning-operator, machine-approver, dns-operator, ingress-operator, community-operators-catalog, kube-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a651d305-0bc4-4e50-8659-237a66cb9f85\n            correlation ID: 1caf4b05-a329-41fc-bcd5-7a64b3705d89\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7279f16aeda941425101f04a4a62215954d6c0260c6e8c861fcfeef38b693a43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"3ec3f52024a46a1f942ae3af19740a4993c313ed0bf4cec5b820ab716098ba7c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"748d80cdff3c2155587af4eebf5a2a6339e8a481299b6bc1aac19a11b8f45f1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"d28602bc01604b20a4fd1810051138a3b4d9daa151d24a925a551647d550c3c4","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa7bf968ac0\u003e: \n    failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-dw92nhksl48m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\",\n      \"name\": \"956158f8-f657-469c-a6c1-31c01e3a3b19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:05.287575354Z\",\n      \"endTime\": \"2026-10-22T02:54:10.367808601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\n            correlation ID: 70395257-9eb1-4aa1-8773-0b99bd5a9b69\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:103]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa7bf968ac0\u003e: \n    failed to create HCP cluster default-ingress-cert: failed waiting for cluster=\"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-dw92nhksl48m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\",\n      \"name\": \"956158f8-f657-469c-a6c1-31c01e3a3b19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:05.287575354Z\",\n      \"endTime\": \"2026-10-22T02:54:10.367808601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/956158f8-f657-469c-a6c1-31c01e3a3b19\n            correlation ID: 70395257-9eb1-4aa1-8773-0b99bd5a9b69\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76b1136d166615c72cbdc4f973e46fbd482810a67d1457271c8f4e0218824453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"66fe9486ebbdab2264fdca019d93b069038bbe27a27a787cb2a4da95a08e65ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"841d9af1b4437d65711b9cbab1dae21fb43950986360466079001ed644f74f91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"f4c91ed9f592165df1c3c27e2fd22c4b93dc93ae321a35275a1f80ee00060e71","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x146ba53e5a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vgggt2t7qcsd\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x146ba6ff790\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vgggt2t7qcsd\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vgggt2t7qcsd, error: context deadline exceeded"}},{"row_id":"894c2618d29b681f24c612aa95e0b8b12bc878d380084e9709468c7a71ed5fbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"3918b5053e2c1b6926508abb7cd5f09e8c960484f7c759d88c7b0673e703991d","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f5c59786c0\u003e: \n    failed to create HCP cluster np-dg-minor-8ks925: failed waiting for cluster=\"np-dg-minor-8ks925\" in resourcegroup=\"rg-np-dg-minor-8ks925-jgl47czbdtkx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"name\": \"3c0ead9e-e84e-4227-ab86-537912304e63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:59.807393048Z\",\n      \"endTime\": \"2026-10-22T03:40:05.155981258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0ead9e-e84e-4227-ab86-537912304e63\n            correlation ID: f292fc5b-a9dd-4382-9a58-48e706cd1073\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8bfaa9cf7180d9a51f7b4ca2cfe809207b24e7e60adbd5064d705f12b9773ff0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"cbeccff849e6332c34aba59d393fdbae4f3ec1001f41174ac4f54440f6912fdd","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d5d614b1333ba162d15badea4f987632d74a7727f196733fd0ec29e6032b67a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"7a1cec7cd36f34b86775ec4e998ac79d4f3902ec309e0df7dd447a697e1ef501","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x133ad1a668c0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6f4gg27gdpsf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\",\n      \"name\": \"100e0456-7bbd-4421-b2b7-99394dc2ddc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.328419317Z\",\n      \"endTime\": \"2026-10-22T02:03:06.759634076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, openshift-controller-manager, cluster-policy-controller, dns-operator, community-operators-catalog, catalog-operator, csi-snapshot-controller-operator, azure-cloud-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, certified-operators-catalog, machine-approver, control-plane-pki-operator, kube-controller-manager, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\n            correlation ID: bee0e436-ae08-4d85-a183-4029b932065b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x133ad1a668c0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6f4gg27gdpsf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\",\n      \"name\": \"100e0456-7bbd-4421-b2b7-99394dc2ddc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.328419317Z\",\n      \"endTime\": \"2026-10-22T02:03:06.759634076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, openshift-controller-manager, cluster-policy-controller, dns-operator, community-operators-catalog, catalog-operator, csi-snapshot-controller-operator, azure-cloud-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, certified-operators-catalog, machine-approver, control-plane-pki-operator, kube-controller-manager, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/100e0456-7bbd-4421-b2b7-99394dc2ddc5\n            correlation ID: bee0e436-ae08-4d85-a183-4029b932065b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8dceddda188e0f7abc122d8951121bdbfd6e76ecb4c201d0b6ff0fb5e5aeadf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"105a43b4bf54437760ddaec80d7274c76e0880fdd04198d3317c96b1e41e0747","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2951a1afed40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hfpdp8lmxh49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"name\": \"0119dd29-201c-4c22-9ebd-7d3a5b8701fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.130621039Z\",\n      \"endTime\": \"2026-10-22T02:03:06.668308317Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-collect-profiles, openshift-apiserver, openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, kube-scheduler, cluster-policy-controller, packageserver, certified-operators-catalog, hosted-cluster-config-operator, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, cluster-network-operator, konnectivity-agent, kube-storage-version-migrator, catalog-operator, cluster-storage-operator, redhat-operators-catalog, cluster-autoscaler, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0119dd29-201c-4c22-9ebd-7d3a5b8701fc\n            correlation ID: 909da2c2-f472-4c31-8509-d71d62f94fb1\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"92153174791b2df69a4ec2a7af0cb85ca4283abdcc67ab7e07ed347999fd00b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"940d1146406ccc6a3297d550ec44a0b5d1a43287589036549f13183fd87e514e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34aa3403e6c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cqgmsxlrrppn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\",\n      \"name\": \"5ee102d0-1601-47d2-8777-14ff5b026dea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:04.925762167Z\",\n      \"endTime\": \"2026-10-22T02:03:06.678876471Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, cluster-storage-operator, cluster-image-registry-operator, certified-operators-catalog, kube-storage-version-migrator, machine-approver, kube-controller-manager, kube-scheduler, openshift-controller-manager, cluster-network-operator, redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, packageserver, cluster-version-operator, ingress-operator, konnectivity-agent, community-operators-catalog, catalog-operator, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\n            correlation ID: 3d6a20c9-050a-45db-a515-be5eb69c9b7e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:147]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34aa3403e6c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cqgmsxlrrppn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\",\n      \"name\": \"5ee102d0-1601-47d2-8777-14ff5b026dea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:04.925762167Z\",\n      \"endTime\": \"2026-10-22T02:03:06.678876471Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, cluster-storage-operator, cluster-image-registry-operator, certified-operators-catalog, kube-storage-version-migrator, machine-approver, kube-controller-manager, kube-scheduler, openshift-controller-manager, cluster-network-operator, redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, packageserver, cluster-version-operator, ingress-operator, konnectivity-agent, community-operators-catalog, catalog-operator, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ee102d0-1601-47d2-8777-14ff5b026dea\n            correlation ID: 3d6a20c9-050a-45db-a515-be5eb69c9b7e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9486eaa8ef7e513a0d131fc59da37de3ec5356307e4bdc947c687784a608ad96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"af774aac99c529ad98d882b83b9a1e38d054f96b4ba76e020f5318718f9b0f61","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39761e268740\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dm68szdn2k7s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\",\n      \"name\": \"3da97f2b-8dc4-47f0-81ba-91f4190a2d02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:26:51.272492265Z\",\n      \"endTime\": \"2026-10-22T02:56:00.377090738Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\n            correlation ID: e08d62dc-1750-4dd8-bfbf-0bd47a4c0c8a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39761e268740\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dm68szdn2k7s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\",\n      \"name\": \"3da97f2b-8dc4-47f0-81ba-91f4190a2d02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:26:51.272492265Z\",\n      \"endTime\": \"2026-10-22T02:56:00.377090738Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3da97f2b-8dc4-47f0-81ba-91f4190a2d02\n            correlation ID: e08d62dc-1750-4dd8-bfbf-0bd47a4c0c8a\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9abf4a68d3b43065e3582e62af3027c4f8ff38818ad3b81ae210632ec8f22850","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"cb6c62fc5bd1672b019609253e5970cf0be960ae29f59adef3f3e3018a703317","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-20-kwjc5xlqs8tk/cluster-candidate-4-20-bcqtqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361f110a43c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bcqtqw: failed waiting for cluster=\"cluster-candidate-4-20-bcqtqw\" in resourcegroup=\"rg-candidate-4-20-kwjc5xlqs8tk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\",\n      \"name\": \"9d6851f9-9f2d-451b-b4e5-4ae16700f297\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:15.911807316Z\",\n      \"endTime\": \"2026-10-22T02:50:20.515370897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-collect-profiles, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, machine-approver, konnectivity-agent, cluster-policy-controller, control-plane-pki-operator, kube-scheduler, openshift-route-controller-manager, dns-operator, ingress-operator, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, cluster-storage-operator, catalog-operator, olm-operator, packageserver, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, community-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\n            correlation ID: a2c7933e-26ff-48cd-92c5-66f72a0f2820\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-20-kwjc5xlqs8tk/cluster-candidate-4-20-bcqtqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361f110a43c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bcqtqw: failed waiting for cluster=\"cluster-candidate-4-20-bcqtqw\" in resourcegroup=\"rg-candidate-4-20-kwjc5xlqs8tk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\",\n      \"name\": \"9d6851f9-9f2d-451b-b4e5-4ae16700f297\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:15.911807316Z\",\n      \"endTime\": \"2026-10-22T02:50:20.515370897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-collect-profiles, cluster-version-operator, cluster-image-registry-operator, cluster-autoscaler, machine-approver, konnectivity-agent, cluster-policy-controller, control-plane-pki-operator, kube-scheduler, openshift-route-controller-manager, dns-operator, ingress-operator, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, cluster-storage-operator, catalog-operator, olm-operator, packageserver, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, community-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9d6851f9-9f2d-451b-b4e5-4ae16700f297\n            correlation ID: a2c7933e-26ff-48cd-92c5-66f72a0f2820\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d3898f386f41b5d2400e8115dc828ad34318943777ec71517ce7143af453432","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"37af00141384f00be88a064e4ee9337863ec4649e69343bc6f3df05230fa74a0","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7938a410140\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ql2jtgg2jsjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"name\": \"65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:38:14.381676151Z\",\n      \"endTime\": \"2026-10-22T02:07:16.666379289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/65c2874a-4e4d-4754-a6c4-c23ee1f34bb9\n            correlation ID: 3892a8ef-862d-414b-aadd-333496c0245e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d7f3e51e5c8944d127d9b0bedbb04995ab9a2142952f40d9113b8b2ddc19461","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"ac79520c264a03cdae111d50487f33410c268ee3536a309598ebaf8fa5495141","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a1ee47c8accf40c96e2a39ba5850d4b65f69783180a0cd4677f9e6f32b9908de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"93b4242fbca78a6804343fa0c780a64cee3b6f7f909e539aa95b885d3208f48b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x144fc2ee5040\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfpbsf629cbt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\",\n      \"name\": \"b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:03.962666335Z\",\n      \"endTime\": \"2026-10-22T02:03:06.792539952Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\n            correlation ID: f2690118-3948-45a7-a4ae-1b177d998957\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x144fc2ee5040\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfpbsf629cbt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\",\n      \"name\": \"b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:03.962666335Z\",\n      \"endTime\": \"2026-10-22T02:03:06.792539952Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b6385dea-fc39-4d5c-bf4a-4517f6ef3e1e\n            correlation ID: f2690118-3948-45a7-a4ae-1b177d998957\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a48a0f06851cdd013019959f06cc52a334eed8f8c46739e4a28d1fddb221f3b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"222ffc9c563b0b4467dc853c34c77c4694cb2ce41c4c51c01cfbc0965f89f195","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-nbxbs6\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24bdb9a64a20\u003e: \n    failed to create HCP cluster np-noedge-nbxbs6: failed waiting for cluster=\"np-noedge-nbxbs6\" in resourcegroup=\"rg-np-noedge-nbxbs6-kqkfkllsq598\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"name\": \"66bec196-bb1d-462f-9d3a-76228415a662\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:43.148572951Z\",\n      \"endTime\": \"2026-10-22T03:39:45.079565863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66bec196-bb1d-462f-9d3a-76228415a662\n            correlation ID: 547b3712-e9f5-410b-87e2-dd2721dd720f\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a4c00a00aee1a146d3d3fa93b9d5f508bee579ec339d16e3cf9d59c0c853bab1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"00e889108d18b2602a9701e81c628e0117c6af2ac6506f48e3dc6377714f2dec","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:86]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12e0378a8e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-s6hb8822zv5x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"name\": \"b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:12.606084954Z\",\n      \"endTime\": \"2026-10-22T02:03:16.732035531Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcfc1d-56f0-4a0e-be27-eb59d78d3391\n            correlation ID: d29a315a-5b37-4a91-af7b-1415cfe4e5ec\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a61d9f5a441c602935455ce9a49026f62432dff238d6c77727fbf8382697e061","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"00c7da51be8e89ce15ce8e10590e25d85b2282252d307b006276577d947b9e86","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd879a7a0b60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7ht4n2l8tbfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"name\": \"94676dae-e02f-4af7-9503-8526a46b136a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:54.675375561Z\",\n      \"endTime\": \"2026-10-22T02:02:56.694987866Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94676dae-e02f-4af7-9503-8526a46b136a\n            correlation ID: eb262c16-35c1-4306-89d7-1ab7a4a6f8e5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a6b5058c7f685f625ba2588e8d51ed17521a04908c0fb1095aaa2e603b7f16d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"d3dedfa37fc8f8f9d48c596be2117ea1ea85b98d4f8b0aba0451aa1f7bea81bc","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e201b18a700\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w4bh6t74lrm8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"name\": \"3ffa3393-14e6-485e-a06e-d34ad16e45af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:33:57.248605332Z\",\n      \"endTime\": \"2026-10-22T02:03:06.666838162Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, olm-operator, openshift-route-controller-manager, cluster-storage-operator, packageserver, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, dns-operator, ingress-operator, catalog-operator, certified-operators-catalog, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ffa3393-14e6-485e-a06e-d34ad16e45af\n            correlation ID: d4851c32-d0b7-4ef1-9333-ecb958420c6c\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a833d4dda644f16114e3c12fc5e369d746be23015c9620a9e9accbdb478c001a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"19248e56dbbe622fc0ab707175608011b12ef937b74bee06e900da1ff17799d8","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-xzrpbt\" with version 4.21.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x913e9d26a60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-xzrpbt: failed waiting for cluster=\"cluster-zstream-4-21-xzrpbt\" in resourcegroup=\"rg-zstream-upgrade-4-21-xgpgfghfg7wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"name\": \"d76aa69d-45fb-439f-889e-97ea3ff34f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:56.334901591Z\",\n      \"endTime\": \"2026-10-22T02:52:00.375914793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, redhat-operators-catalog, certified-operators-catalog, control-plane-pki-operator, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, cluster-autoscaler, hosted-cluster-config-operator, olm-operator, packageserver, ingress-operator, cluster-image-registry-operator, machine-approver, cluster-policy-controller, cluster-node-tuning-operator, konnectivity-agent, kube-controller-manager, openshift-controller-manager, cluster-network-operator, catalog-operator, community-operators-catalog, kube-scheduler, openshift-apiserver, csi-snapshot-controller-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d76aa69d-45fb-439f-889e-97ea3ff34f99\n            correlation ID: b1fc9a13-2fc1-4373-b2a1-ce2064832e2e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aaa44b2e6fc7e3f5b9b65e1c83aae9d7d245075c51bba212a9b70ea5ebbfefe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"7532d5417c94c768b19029dc4b5188f6d4f7b0525e268ed507fdd211a7691731","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x115fb34512c0\u003e: \n    failed to create HCP cluster np-downgrade-42l48k: failed waiting for cluster=\"np-downgrade-42l48k\" in resourcegroup=\"rg-np-downgrade-42l48k-5zqzd6q4wnms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\",\n      \"name\": \"05348ada-50c8-472d-8c3c-a7bf34489fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:02.734882718Z\",\n      \"endTime\": \"2026-10-22T03:40:05.149727722Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-policy-controller, cluster-image-registry-operator, packageserver, control-plane-pki-operator, dns-operator, openshift-apiserver, cluster-node-tuning-operator, kube-controller-manager, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, olm-collect-profiles, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-autoscaler, ingress-operator, redhat-marketplace-catalog, catalog-operator, kube-scheduler, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\n            correlation ID: 513d2a7a-52d7-4655-b715-b68c051bb6dc\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x115fb34512c0\u003e: \n    failed to create HCP cluster np-downgrade-42l48k: failed waiting for cluster=\"np-downgrade-42l48k\" in resourcegroup=\"rg-np-downgrade-42l48k-5zqzd6q4wnms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\",\n      \"name\": \"05348ada-50c8-472d-8c3c-a7bf34489fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:02.734882718Z\",\n      \"endTime\": \"2026-10-22T03:40:05.149727722Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-policy-controller, cluster-image-registry-operator, packageserver, control-plane-pki-operator, dns-operator, openshift-apiserver, cluster-node-tuning-operator, kube-controller-manager, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, olm-collect-profiles, azure-cloud-controller-manager, csi-snapshot-controller-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-autoscaler, ingress-operator, redhat-marketplace-catalog, catalog-operator, kube-scheduler, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/05348ada-50c8-472d-8c3c-a7bf34489fb8\n            correlation ID: 513d2a7a-52d7-4655-b715-b68c051bb6dc\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bca953eed3f2b89b57aaf2e70dbed940290cbcf23b020076c47d07581a7436aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"ba521c2c19919a5caa070da355f36879cb81eb48d013d05ff609671940da6a95","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x331e8e5a0000\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-vj9q24f28wn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\",\n      \"name\": \"d012ea11-280a-4508-8665-b98bf45a8243\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:30.550148133Z\",\n      \"endTime\": \"2026-10-22T02:54:30.366190637Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\n            correlation ID: c24dd16b-dee6-4cf8-8524-5a4ca58ee6d0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x331e8e5a0000\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-vj9q24f28wn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\",\n      \"name\": \"d012ea11-280a-4508-8665-b98bf45a8243\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:25:30.550148133Z\",\n      \"endTime\": \"2026-10-22T02:54:30.366190637Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d012ea11-280a-4508-8665-b98bf45a8243\n            correlation ID: c24dd16b-dee6-4cf8-8524-5a4ca58ee6d0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bec1b246257d6546285a2a6753761b9784171e790c024b260757d0ce0575b079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"3dd0847594fe9934ecaef404cfd5b2e0d30bdd83e3bdd0629d37df342bb8fe04","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ad1958370e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6z47gdxtfxk2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\",\n      \"name\": \"88d7d257-b7b4-4fd6-968b-d958c1947661\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:11.887412502Z\",\n      \"endTime\": \"2026-10-22T03:38:15.157786729Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, certified-operators-catalog, dns-operator, catalog-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, community-operators-catalog, olm-operator, packageserver, openshift-route-controller-manager, kube-controller-manager, kube-scheduler, csi-snapshot-controller-operator, openshift-controller-manager, hosted-cluster-config-operator, redhat-marketplace-catalog, control-plane-pki-operator, ingress-operator, konnectivity-agent, machine-approver, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\n            correlation ID: 20139e98-2ffa-42ab-b589-fa9d706c6e08\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ad1958370e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6z47gdxtfxk2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\",\n      \"name\": \"88d7d257-b7b4-4fd6-968b-d958c1947661\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:11.887412502Z\",\n      \"endTime\": \"2026-10-22T03:38:15.157786729Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, certified-operators-catalog, dns-operator, catalog-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, community-operators-catalog, olm-operator, packageserver, openshift-route-controller-manager, kube-controller-manager, kube-scheduler, csi-snapshot-controller-operator, openshift-controller-manager, hosted-cluster-config-operator, redhat-marketplace-catalog, control-plane-pki-operator, ingress-operator, konnectivity-agent, machine-approver, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88d7d257-b7b4-4fd6-968b-d958c1947661\n            correlation ID: 20139e98-2ffa-42ab-b589-fa9d706c6e08\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bf8e98232622e8ef016c07b8f597b4d28f031d727111a509cd7ce64f53ef92b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"78bb0fd5dc8b2957b03be693a8b15a24b0624bf8afc1be3e13738a6518bfff22","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nnvzdg with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2bd5ec4c4840\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-nnvzdg: failed waiting for cluster=\"np-version-upgrade-cluster-nnvzdg\" in resourcegroup=\"rg-np-version-upgrade-nnvzdg-mqmfl8g2qxgk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\",\n      \"name\": \"928f3c23-211a-4f82-842c-98d26e84d456\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:47.131790254Z\",\n      \"endTime\": \"2026-10-22T03:38:55.090650407Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, csi-snapshot-controller-operator, certified-operators-catalog, machine-approver, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-marketplace-catalog, community-operators-catalog, catalog-operator, kube-scheduler, openshift-apiserver, olm-collect-profiles, ingress-operator, cluster-version-operator, kube-controller-manager, cluster-storage-operator, packageserver, olm-operator, cluster-autoscaler, dns-operator, openshift-controller-manager, control-plane-pki-operator, cluster-network-operator, redhat-operators-catalog, cluster-node-tuning-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\n            correlation ID: 0bf299e5-3230-4f67-8cf1-6944f93fcb2e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nnvzdg with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2bd5ec4c4840\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-nnvzdg: failed waiting for cluster=\"np-version-upgrade-cluster-nnvzdg\" in resourcegroup=\"rg-np-version-upgrade-nnvzdg-mqmfl8g2qxgk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\",\n      \"name\": \"928f3c23-211a-4f82-842c-98d26e84d456\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:47.131790254Z\",\n      \"endTime\": \"2026-10-22T03:38:55.090650407Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, csi-snapshot-controller-operator, certified-operators-catalog, machine-approver, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-marketplace-catalog, community-operators-catalog, catalog-operator, kube-scheduler, openshift-apiserver, olm-collect-profiles, ingress-operator, cluster-version-operator, kube-controller-manager, cluster-storage-operator, packageserver, olm-operator, cluster-autoscaler, dns-operator, openshift-controller-manager, control-plane-pki-operator, cluster-network-operator, redhat-operators-catalog, cluster-node-tuning-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/928f3c23-211a-4f82-842c-98d26e84d456\n            correlation ID: 0bf299e5-3230-4f67-8cf1-6944f93fcb2e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c4e8864a3455278ca7e8a1e2e7233a194835826802c4fbe6f1fbb28d179ec70c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"a705768924b8c91914dfee887fd041226da8b0d2a7c49c71af1ef216422bf744","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f4cc33f120\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-vwdnmjj8rtd5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\",\n      \"name\": \"859603f2-b753-4e20-87c5-5cd7720f1d8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:07.162285549Z\",\n      \"endTime\": \"2026-10-22T02:03:16.736233168Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\n            correlation ID: 0121aed1-37e4-4ef9-9b80-ddb831aa669a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f4cc33f120\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-vwdnmjj8rtd5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\",\n      \"name\": \"859603f2-b753-4e20-87c5-5cd7720f1d8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:07.162285549Z\",\n      \"endTime\": \"2026-10-22T02:03:16.736233168Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/859603f2-b753-4e20-87c5-5cd7720f1d8e\n            correlation ID: 0121aed1-37e4-4ef9-9b80-ddb831aa669a\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c8229c63c0df04ea4c94ca886a73526fbb6eac10f00a435f4eb1e0f4933a9129","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"6d20fac6b2c664e06dd79b23d104b0707d6bcfcfba1dad3f3a5feeea9f41e97d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vwsld8 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc02210a0420\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vwsld8: failed waiting for cluster=\"np-version-upgrade-cluster-vwsld8\" in resourcegroup=\"rg-np-version-upgrade-vwsld8-b78v68n9cqj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"name\": \"7061ec27-2552-401b-a8b5-5bcf7cc27448\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:10:01.822651936Z\",\n      \"endTime\": \"2026-10-22T03:39:05.079618983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, hosted-cluster-config-operator, cluster-image-registry-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, openshift-controller-manager, packageserver, cluster-policy-controller, dns-operator, olm-operator, cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, konnectivity-agent, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7061ec27-2552-401b-a8b5-5bcf7cc27448\n            correlation ID: 8d4a80f0-34d4-471d-b30b-2c4ee315636d\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cba44d670c8c67b020691c88ff67d4cc9156a24d3262ac2e32e8b57362d25f58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"b542846af6df57de6bbea4872c890fa54411aeb1201b4a017c6a8bfb54ce2c21","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x85472092ba0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d4m4wrpd9cv2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\",\n      \"name\": \"8ed19878-5858-4619-b204-f2d81fa433c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:56.765676016Z\",\n      \"endTime\": \"2026-10-22T02:04:56.661581883Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\n            correlation ID: 6f084968-c4fe-49c7-b4c8-bb07b85c93c8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x85472092ba0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d4m4wrpd9cv2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\",\n      \"name\": \"8ed19878-5858-4619-b204-f2d81fa433c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:56.765676016Z\",\n      \"endTime\": \"2026-10-22T02:04:56.661581883Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ed19878-5858-4619-b204-f2d81fa433c2\n            correlation ID: 6f084968-c4fe-49c7-b4c8-bb07b85c93c8\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cca19a59e30229c0353fddd90dc1e1a45128c47dd29f25285b61ce4b9a15e8db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"89edbfd547763da1be2ecb5ac3a37c254a734ffe88e2c8157270658bdc6cd116","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f84045ab20\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pbcvbtl256h6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\",\n      \"name\": \"2754f526-7d96-46a2-b85d-5f23a12e42c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.419619024Z\",\n      \"endTime\": \"2026-10-22T02:03:36.794767033Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-controller-manager, packageserver, machine-approver, cluster-policy-controller, konnectivity-agent, certified-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, olm-operator, olm-collect-profiles, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, openshift-controller-manager, openshift-route-controller-manager, kube-scheduler, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, openshift-apiserver, cluster-version-operator, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\n            correlation ID: 65497397-2503-43f4-9c67-ed34b0edfa0b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f84045ab20\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pbcvbtl256h6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\",\n      \"name\": \"2754f526-7d96-46a2-b85d-5f23a12e42c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.419619024Z\",\n      \"endTime\": \"2026-10-22T02:03:36.794767033Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-controller-manager, packageserver, machine-approver, cluster-policy-controller, konnectivity-agent, certified-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, olm-operator, olm-collect-profiles, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, openshift-controller-manager, openshift-route-controller-manager, kube-scheduler, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, openshift-apiserver, cluster-version-operator, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2754f526-7d96-46a2-b85d-5f23a12e42c5\n            correlation ID: 65497397-2503-43f4-9c67-ed34b0edfa0b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd0cae4e05ddb1dfb3589913ab58ae1a4d65a387b618e92a033ce06848ebc2ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"9d48f25db5125a951413a47642b0efecb6d5d06c908e8b1e5fde16b1b20e1440","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28c6381f07c0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-plwcnbx7hnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"name\": \"5f96f74a-1c0c-4295-a2c6-9dce177af3f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:31.83333937Z\",\n      \"endTime\": \"2026-10-22T02:03:36.684949695Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m44s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96f74a-1c0c-4295-a2c6-9dce177af3f9\n            correlation ID: aad1d81c-32be-4413-84a9-7ac0b5dcc1a8\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d1c13701e0d4e35c93f0093655fcf1aa089e49d96fa046c10ebddd29bd423897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"43ca0ef5137db4c92687ca44af2e5269c14f5a14f38084e6f18c59c868e18577","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dbab156eb8d1af22daef94c1ad10a32e7da4395c00bccb4aa8ac164da5ea737a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"f052c83930df14da60956d205c94d7b6903bdbd091c343818e9c8edabac6c7c6","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x353e84400c40\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mmwcq8j6x77j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\",\n      \"name\": \"045880bb-951e-46ab-b5a0-e93973ee8401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.474310765Z\",\n      \"endTime\": \"2026-10-22T02:03:06.762878168Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-policy-controller, dns-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, control-plane-pki-operator, kube-scheduler, openshift-controller-manager, cluster-network-operator, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, hosted-cluster-config-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, machine-approver, kube-controller-manager, openshift-route-controller-manager, azure-cloud-controller-manager, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\n            correlation ID: 1f9e7a18-e51b-4413-80b7-a809b5d0f849\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x353e84400c40\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mmwcq8j6x77j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\",\n      \"name\": \"045880bb-951e-46ab-b5a0-e93973ee8401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.474310765Z\",\n      \"endTime\": \"2026-10-22T02:03:06.762878168Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-policy-controller, dns-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, control-plane-pki-operator, kube-scheduler, openshift-controller-manager, cluster-network-operator, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, hosted-cluster-config-operator, catalog-operator, cluster-storage-operator, csi-snapshot-controller-operator, konnectivity-agent, machine-approver, kube-controller-manager, openshift-route-controller-manager, azure-cloud-controller-manager, olm-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045880bb-951e-46ab-b5a0-e93973ee8401\n            correlation ID: 1f9e7a18-e51b-4413-80b7-a809b5d0f849\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df8315b1e791d9f2be5acce16705a1c7d0bbd365d11341828641145e3aff11f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"a6864331b74f9520caaa01fb142d42bd934a1b337ffea6fcc51fd907a468b304","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-tvqgcd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a1b74a0c6e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-tvqgcd: failed waiting for cluster=\"cluster-zstream-4-23-tvqgcd\" in resourcegroup=\"rg-zstream-upgrade-4-23-z4qsqrdw854d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"name\": \"bdede344-c320-476c-93e8-c3141c6d343a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:59.55049493Z\",\n      \"endTime\": \"2026-10-22T02:52:00.596740917Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, openshift-controller-manager, cluster-network-operator, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, dns-operator, community-operators-catalog, olm-operator, cluster-image-registry-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bdede344-c320-476c-93e8-c3141c6d343a\n            correlation ID: 946c8b3d-71c1-46d3-ad96-760f309b6eb8\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e2b439034e27312e21ec10607fcb38a7849a49f6e4515b5b269ad8dfe26c654f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"610533cebef7feda8d458dc8e967f9408265cfa9d327e5638860a65b8ee27823","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x327c31abadc0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vzj54nlw5csp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"name\": \"f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:35:36.727726764Z\",\n      \"endTime\": \"2026-10-22T02:04:36.824596885Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9fa1671-b5d8-42db-9ac9-d89b520cbbf9\n            correlation ID: ba4affaa-364d-40fe-b039-d27e78c0a5e0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e3eaa56ae3b8e6a796ab8e06aed9ec06ba8b8f288bd9783e3a0cbd5d5b7701ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"e50814639bbed4fd159acc63cc76eb3b3ea44eedc3c0972c63ffedfeeb1d6b35","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbfee40279a0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-fx2f2r72cdzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\",\n      \"name\": \"157c737e-923c-42ef-b1d7-edf1ae3eb633\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:17.250629196Z\",\n      \"endTime\": \"2026-10-22T02:03:26.781048315Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-image-registry-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, catalog-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-version-operator, dns-operator, ingress-operator, packageserver, machine-approver, openshift-apiserver, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog, cluster-policy-controller, cluster-network-operator, olm-collect-profiles, cluster-autoscaler, kube-scheduler, openshift-route-controller-manager, olm-operator, community-operators-catalog, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\n            correlation ID: e8e05aa0-c13d-42e2-915a-25e9e6211e06\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbfee40279a0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-fx2f2r72cdzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\",\n      \"name\": \"157c737e-923c-42ef-b1d7-edf1ae3eb633\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:17.250629196Z\",\n      \"endTime\": \"2026-10-22T02:03:26.781048315Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-image-registry-operator, openshift-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, catalog-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-version-operator, dns-operator, ingress-operator, packageserver, machine-approver, openshift-apiserver, csi-snapshot-controller-operator, konnectivity-agent, certified-operators-catalog, cluster-policy-controller, cluster-network-operator, olm-collect-profiles, cluster-autoscaler, kube-scheduler, openshift-route-controller-manager, olm-operator, community-operators-catalog, kube-controller-manager; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/157c737e-923c-42ef-b1d7-edf1ae3eb633\n            correlation ID: e8e05aa0-c13d-42e2-915a-25e9e6211e06\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e4202c5a2fbc0249de5ef757627f1a1cdd549e815ad79438b4b92b43b3d51c04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"fd02807286b5b83c10835eef9428f86b7fe1936bd74115749fc73fc5111727ba","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-zj4mcv\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x336a41a7c8a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-zj4mcv: failed waiting for cluster=\"cluster-zstream-4-20-zj4mcv\" in resourcegroup=\"rg-zstream-upgrade-4-20-2z67w2qcbxs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"name\": \"03c686a3-8b51-4273-9c18-ce6884c5a033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:22:44.492283349Z\",\n      \"endTime\": \"2026-10-22T02:51:50.526771089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-storage-operator, cluster-image-registry-operator, catalog-operator, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, dns-operator, redhat-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-route-controller-manager, ingress-operator, olm-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/03c686a3-8b51-4273-9c18-ce6884c5a033\n            correlation ID: 5cab7c5c-8f8d-4763-8268-cddbff9d06f5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e5b1dbec48b0e46e52eeeca7ba459a4b2976cb9c4748adad0febc7c815a6cf76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"74f59f951658f16f8d54e8163c072eeb6209baea5d1369c72844e0d3da80a25d","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hlfqz4 with version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26fe4f09e7e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hlfqz4: failed waiting for cluster=\"np-version-upgrade-cluster-hlfqz4\" in resourcegroup=\"rg-np-version-upgrade-hlfqz4-hxtt44szpfn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"name\": \"9b1bdec9-cda2-4c03-8af1-a62744ae5bac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:09:57.381547287Z\",\n      \"endTime\": \"2026-10-22T03:39:05.056394482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, redhat-operators-catalog, catalog-operator, cluster-version-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-policy-controller, cluster-storage-operator, openshift-route-controller-manager, kube-controller-manager, ingress-operator, packageserver, olm-operator, kube-scheduler, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9b1bdec9-cda2-4c03-8af1-a62744ae5bac\n            correlation ID: ae3669ea-d6b2-47bf-90e6-616390cf31a2\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e7214d2b3871c293eb51937e26590b178f4002d8c82cff9edd9b7543bfa2a3bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"96e526637370066fb982529359f8fb2c1357e11506ca6d9024d12db800540150","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"eec4049bd6bac16f6b303255ffc5b16a07d72da802c574a6e5d2bd07bcabbc6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"d15e0ee9a765947a51994971a21dbf7f7f640ff3a8e13f8d501e97316301c1e6","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5228e164e00\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-d2fn5kshsl75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\",\n      \"name\": \"e2b2e462-7fbf-427d-acfa-1649fda7fc18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:10.501548627Z\",\n      \"endTime\": \"2026-10-22T03:40:15.062794425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, konnectivity-agent, ingress-operator, openshift-route-controller-manager, cluster-network-operator, kube-controller-manager, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, openshift-controller-manager, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, dns-operator, redhat-marketplace-catalog, kube-scheduler, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\n            correlation ID: ae820045-80be-4efc-9077-32c6901a632b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5228e164e00\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-d2fn5kshsl75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\",\n      \"name\": \"e2b2e462-7fbf-427d-acfa-1649fda7fc18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:10.501548627Z\",\n      \"endTime\": \"2026-10-22T03:40:15.062794425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, konnectivity-agent, ingress-operator, openshift-route-controller-manager, cluster-network-operator, kube-controller-manager, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, openshift-controller-manager, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, dns-operator, redhat-marketplace-catalog, kube-scheduler, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e2b2e462-7fbf-427d-acfa-1649fda7fc18\n            correlation ID: ae820045-80be-4efc-9077-32c6901a632b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"eefd31193ce7583337b26c7bb05bc18f381677fb6c90c3fa6187b1ce2811ed30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"012587d7d4630c81e42093fd6ab2c1257951217e2525d7784fd89a2e89112676","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19243183a940\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-df27xfnfzvrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\",\n      \"name\": \"92eda184-8a06-461e-9d73-222caa436db8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T01:34:06.303940712Z\",\n      \"endTime\": \"2026-10-22T02:03:06.782181855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, capi-provider, azure-cloud-controller-manager, csi-snapshot-controller-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, machine-approver, cluster-network-operator, konnectivity-agent, redhat-marketplace-catalog, olm-operator, packageserver, cluster-policy-controller, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92eda184-8a06-461e-9d73-222caa436db8\n            correlation ID: 8174d018-ca0b-4942-ba1c-716f164824c9\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f0f3370c18d67b0dca94bbd758dcac98bfd5cc14f3253422a275a703d22711b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"8181e1d728fd2f9d65a27b1a6f385b120f5a9c516932134b128d158f80fe37e2","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-q2kkkp\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x300bf5b7a840\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-q2kkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-q2kkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t4mn7sf29xrr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"name\": \"54a4385e-0b42-4987-acf3-1b0207b030a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:48.751941498Z\",\n      \"endTime\": \"2026-10-22T02:52:50.375994252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54a4385e-0b42-4987-acf3-1b0207b030a5\n            correlation ID: f3ca853a-64dc-4e52-ab57-050647054f81\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f4b7f8c4a26e5f697f4bed90128ef7efe9abd83a9461f757be7c8ae3fa7c3cbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"10bb0bdb748cd2f51cfdd05065854a330965cf2734479264377206588e187b5c","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x262004875180\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-v4w4cbtgl6st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"name\": \"0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:27:27.45827201Z\",\n      \"endTime\": \"2026-10-22T02:56:30.386782585Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ffe01b2-d2c4-4e16-a6d6-042cbbeb51ec\n            correlation ID: a0e42f8f-bf63-4670-b77c-e6ba06f19444\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fa727a8a1227b87a24cb79b550724bad5ed2302ba9a64e8b63ab6cf5a2c90d54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","failed_at":"e2e","signature_id":"c0bc7ce2f8badc2267d7a65ad1002f4d46160b2c3b4087c1135728e23c2b816e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7pjtrh\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fc646288640\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7pjtrh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7pjtrh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ph2qs5rvpjbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\",\n      \"name\": \"1952a1f5-b390-468e-8207-de9b42085d84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:50.650035503Z\",\n      \"endTime\": \"2026-10-22T02:52:50.371348301Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\n            correlation ID: 13cd48b6-3df3-4a5e-9127-3bc8b9f3564d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7pjtrh\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fc646288640\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7pjtrh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7pjtrh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ph2qs5rvpjbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\",\n      \"name\": \"1952a1f5-b390-468e-8207-de9b42085d84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:50.650035503Z\",\n      \"endTime\": \"2026-10-22T02:52:50.371348301Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1952a1f5-b390-468e-8207-de9b42085d84\n            correlation ID: 13cd48b6-3df3-4a5e-9127-3bc8b9f3564d\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":62,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","fffa36f85bbbff9b9e35ac04e5e1995d6605346e9b86d9872df10486fda0e14f"],"matched_occurrences":62,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}